2024-11-21 21:14:07,513 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-21 21:14:07,529 main DEBUG Took 0.013766 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-21 21:14:07,530 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-21 21:14:07,530 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-21 21:14:07,532 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-21 21:14:07,533 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 21:14:07,542 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-21 21:14:07,555 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 21:14:07,556 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 21:14:07,556 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 21:14:07,557 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 21:14:07,557 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 21:14:07,558 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 21:14:07,559 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 21:14:07,559 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 21:14:07,560 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 21:14:07,560 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 21:14:07,561 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 21:14:07,561 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 21:14:07,562 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 21:14:07,562 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 21:14:07,563 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 21:14:07,563 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 21:14:07,563 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 21:14:07,564 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 21:14:07,564 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 21:14:07,564 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 21:14:07,565 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 21:14:07,565 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 21:14:07,565 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 21:14:07,566 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 21:14:07,566 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 21:14:07,566 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-21 21:14:07,568 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 21:14:07,569 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-21 21:14:07,571 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-21 21:14:07,571 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-21 21:14:07,572 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-21 21:14:07,572 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-21 21:14:07,580 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-21 21:14:07,582 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-21 21:14:07,584 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-21 21:14:07,584 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-21 21:14:07,585 main DEBUG createAppenders(={Console}) 2024-11-21 21:14:07,585 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-21 21:14:07,586 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-21 21:14:07,586 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-21 21:14:07,586 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-21 21:14:07,587 main DEBUG OutputStream closed 2024-11-21 21:14:07,587 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-21 21:14:07,587 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-21 21:14:07,588 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-21 21:14:07,657 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-21 21:14:07,659 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-21 21:14:07,660 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-21 21:14:07,661 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-21 21:14:07,661 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-21 21:14:07,662 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-21 21:14:07,662 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-21 21:14:07,662 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-21 21:14:07,663 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-21 21:14:07,663 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-21 21:14:07,663 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-21 21:14:07,664 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-21 21:14:07,664 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-21 21:14:07,664 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-21 21:14:07,665 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-21 21:14:07,665 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-21 21:14:07,665 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-21 21:14:07,666 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-21 21:14:07,668 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-21 21:14:07,669 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-21 21:14:07,669 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-21 21:14:07,670 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-21T21:14:07,921 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51952d30-d1ef-b0b3-2459-5ab0f09ed1d1 2024-11-21 21:14:07,923 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-21 21:14:07,924 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-21T21:14:07,931 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-21T21:14:07,951 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-21T21:14:07,966 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51952d30-d1ef-b0b3-2459-5ab0f09ed1d1/cluster_d7e18f9d-61ff-69fb-e450-b3a2cf11ddfa, deleteOnExit=true 2024-11-21T21:14:07,966 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-21T21:14:07,967 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51952d30-d1ef-b0b3-2459-5ab0f09ed1d1/test.cache.data in system properties and HBase conf 2024-11-21T21:14:07,968 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51952d30-d1ef-b0b3-2459-5ab0f09ed1d1/hadoop.tmp.dir in system properties and HBase conf 2024-11-21T21:14:07,969 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51952d30-d1ef-b0b3-2459-5ab0f09ed1d1/hadoop.log.dir in system properties and HBase conf 2024-11-21T21:14:07,970 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51952d30-d1ef-b0b3-2459-5ab0f09ed1d1/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-21T21:14:07,970 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51952d30-d1ef-b0b3-2459-5ab0f09ed1d1/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-21T21:14:07,971 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-21T21:14:08,052 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-21T21:14:08,133 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T21:14:08,137 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51952d30-d1ef-b0b3-2459-5ab0f09ed1d1/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-21T21:14:08,138 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51952d30-d1ef-b0b3-2459-5ab0f09ed1d1/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-21T21:14:08,138 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51952d30-d1ef-b0b3-2459-5ab0f09ed1d1/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-21T21:14:08,139 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51952d30-d1ef-b0b3-2459-5ab0f09ed1d1/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-21T21:14:08,139 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51952d30-d1ef-b0b3-2459-5ab0f09ed1d1/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-21T21:14:08,140 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51952d30-d1ef-b0b3-2459-5ab0f09ed1d1/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-21T21:14:08,140 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51952d30-d1ef-b0b3-2459-5ab0f09ed1d1/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-21T21:14:08,141 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51952d30-d1ef-b0b3-2459-5ab0f09ed1d1/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-21T21:14:08,141 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51952d30-d1ef-b0b3-2459-5ab0f09ed1d1/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-21T21:14:08,142 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51952d30-d1ef-b0b3-2459-5ab0f09ed1d1/nfs.dump.dir in system properties and HBase conf 2024-11-21T21:14:08,142 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51952d30-d1ef-b0b3-2459-5ab0f09ed1d1/java.io.tmpdir in system properties and HBase conf 2024-11-21T21:14:08,142 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51952d30-d1ef-b0b3-2459-5ab0f09ed1d1/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-21T21:14:08,143 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51952d30-d1ef-b0b3-2459-5ab0f09ed1d1/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-21T21:14:08,143 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51952d30-d1ef-b0b3-2459-5ab0f09ed1d1/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-21T21:14:09,231 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-21T21:14:09,299 INFO [Time-limited test {}] log.Log(170): Logging initialized @2483ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-21T21:14:09,366 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T21:14:09,426 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-21T21:14:09,446 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-21T21:14:09,446 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-21T21:14:09,447 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-21T21:14:09,459 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T21:14:09,463 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51952d30-d1ef-b0b3-2459-5ab0f09ed1d1/hadoop.log.dir/,AVAILABLE} 2024-11-21T21:14:09,464 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-21T21:14:09,640 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51952d30-d1ef-b0b3-2459-5ab0f09ed1d1/java.io.tmpdir/jetty-localhost-33039-hadoop-hdfs-3_4_1-tests_jar-_-any-10211694020293964704/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-21T21:14:09,651 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:33039} 2024-11-21T21:14:09,651 INFO [Time-limited test {}] server.Server(415): Started @2836ms 2024-11-21T21:14:10,322 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T21:14:10,331 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-21T21:14:10,332 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-21T21:14:10,332 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-21T21:14:10,333 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-21T21:14:10,333 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51952d30-d1ef-b0b3-2459-5ab0f09ed1d1/hadoop.log.dir/,AVAILABLE} 2024-11-21T21:14:10,334 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-21T21:14:10,447 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51952d30-d1ef-b0b3-2459-5ab0f09ed1d1/java.io.tmpdir/jetty-localhost-44775-hadoop-hdfs-3_4_1-tests_jar-_-any-14733930039791657880/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-21T21:14:10,448 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:44775} 2024-11-21T21:14:10,448 INFO [Time-limited test {}] server.Server(415): Started @3633ms 2024-11-21T21:14:10,500 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-21T21:14:11,498 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51952d30-d1ef-b0b3-2459-5ab0f09ed1d1/cluster_d7e18f9d-61ff-69fb-e450-b3a2cf11ddfa/data/data2/current/BP-1102385231-172.17.0.2-1732223648686/current, will proceed with Du for space computation calculation, 2024-11-21T21:14:11,498 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51952d30-d1ef-b0b3-2459-5ab0f09ed1d1/cluster_d7e18f9d-61ff-69fb-e450-b3a2cf11ddfa/data/data1/current/BP-1102385231-172.17.0.2-1732223648686/current, will proceed with Du for space computation calculation, 2024-11-21T21:14:11,525 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-21T21:14:11,571 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb0b8c5bb994a9d62 with lease ID 0xd2d545d35f1909c1: Processing first storage report for DS-91989f13-82a6-43fa-8cfb-bce78569bdbd from datanode DatanodeRegistration(127.0.0.1:35127, datanodeUuid=6f411320-8877-4cf1-8745-6c5208a97a0a, infoPort=36345, infoSecurePort=0, ipcPort=37675, storageInfo=lv=-57;cid=testClusterID;nsid=908215587;c=1732223648686) 2024-11-21T21:14:11,572 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb0b8c5bb994a9d62 with lease ID 0xd2d545d35f1909c1: from storage DS-91989f13-82a6-43fa-8cfb-bce78569bdbd node DatanodeRegistration(127.0.0.1:35127, datanodeUuid=6f411320-8877-4cf1-8745-6c5208a97a0a, infoPort=36345, infoSecurePort=0, ipcPort=37675, storageInfo=lv=-57;cid=testClusterID;nsid=908215587;c=1732223648686), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-21T21:14:11,573 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb0b8c5bb994a9d62 with lease ID 0xd2d545d35f1909c1: Processing first storage report for DS-2635a673-992e-460a-a3e1-796fee4879e2 from datanode DatanodeRegistration(127.0.0.1:35127, datanodeUuid=6f411320-8877-4cf1-8745-6c5208a97a0a, infoPort=36345, infoSecurePort=0, ipcPort=37675, storageInfo=lv=-57;cid=testClusterID;nsid=908215587;c=1732223648686) 2024-11-21T21:14:11,573 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb0b8c5bb994a9d62 with lease ID 0xd2d545d35f1909c1: from storage DS-2635a673-992e-460a-a3e1-796fee4879e2 node DatanodeRegistration(127.0.0.1:35127, datanodeUuid=6f411320-8877-4cf1-8745-6c5208a97a0a, infoPort=36345, infoSecurePort=0, ipcPort=37675, storageInfo=lv=-57;cid=testClusterID;nsid=908215587;c=1732223648686), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-21T21:14:11,606 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51952d30-d1ef-b0b3-2459-5ab0f09ed1d1 2024-11-21T21:14:11,672 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51952d30-d1ef-b0b3-2459-5ab0f09ed1d1/cluster_d7e18f9d-61ff-69fb-e450-b3a2cf11ddfa/zookeeper_0, clientPort=52266, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51952d30-d1ef-b0b3-2459-5ab0f09ed1d1/cluster_d7e18f9d-61ff-69fb-e450-b3a2cf11ddfa/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51952d30-d1ef-b0b3-2459-5ab0f09ed1d1/cluster_d7e18f9d-61ff-69fb-e450-b3a2cf11ddfa/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-21T21:14:11,683 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52266 2024-11-21T21:14:11,697 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T21:14:11,699 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T21:14:11,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741825_1001 (size=7) 2024-11-21T21:14:12,325 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f with version=8 2024-11-21T21:14:12,326 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/hbase-staging 2024-11-21T21:14:12,410 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-21T21:14:12,722 INFO [Time-limited test {}] client.ConnectionUtils(128): master/849dee24a242:0 server-side Connection retries=45 2024-11-21T21:14:12,737 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T21:14:12,737 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T21:14:12,744 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T21:14:12,744 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T21:14:12,744 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T21:14:12,945 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-21T21:14:13,013 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-21T21:14:13,022 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-21T21:14:13,026 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T21:14:13,049 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 4671 (auto-detected) 2024-11-21T21:14:13,050 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-21T21:14:13,069 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42485 2024-11-21T21:14:13,088 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42485 connecting to ZooKeeper ensemble=127.0.0.1:52266 2024-11-21T21:14:13,189 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:424850x0, quorum=127.0.0.1:52266, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T21:14:13,192 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42485-0x1015f3f4af00000 connected 2024-11-21T21:14:13,300 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T21:14:13,302 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T21:14:13,316 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42485-0x1015f3f4af00000, quorum=127.0.0.1:52266, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T21:14:13,321 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f, hbase.cluster.distributed=false 2024-11-21T21:14:13,355 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42485-0x1015f3f4af00000, quorum=127.0.0.1:52266, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T21:14:13,363 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42485 2024-11-21T21:14:13,364 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42485 2024-11-21T21:14:13,367 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42485 2024-11-21T21:14:13,372 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42485 2024-11-21T21:14:13,373 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42485 2024-11-21T21:14:13,503 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/849dee24a242:0 server-side Connection retries=45 2024-11-21T21:14:13,505 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T21:14:13,505 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T21:14:13,506 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T21:14:13,506 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T21:14:13,506 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T21:14:13,508 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-21T21:14:13,512 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T21:14:13,513 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41213 2024-11-21T21:14:13,516 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41213 connecting to ZooKeeper ensemble=127.0.0.1:52266 2024-11-21T21:14:13,518 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T21:14:13,523 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T21:14:13,544 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:412130x0, quorum=127.0.0.1:52266, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T21:14:13,545 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41213-0x1015f3f4af00001 connected 2024-11-21T21:14:13,545 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41213-0x1015f3f4af00001, quorum=127.0.0.1:52266, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T21:14:13,549 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-21T21:14:13,556 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-21T21:14:13,559 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41213-0x1015f3f4af00001, quorum=127.0.0.1:52266, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-21T21:14:13,563 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41213-0x1015f3f4af00001, quorum=127.0.0.1:52266, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T21:14:13,564 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41213 2024-11-21T21:14:13,564 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41213 2024-11-21T21:14:13,565 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41213 2024-11-21T21:14:13,566 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41213 2024-11-21T21:14:13,566 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41213 2024-11-21T21:14:13,580 DEBUG [M:0;849dee24a242:42485 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;849dee24a242:42485 2024-11-21T21:14:13,581 INFO [master/849dee24a242:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/849dee24a242,42485,1732223652547 2024-11-21T21:14:13,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42485-0x1015f3f4af00000, quorum=127.0.0.1:52266, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T21:14:13,597 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41213-0x1015f3f4af00001, quorum=127.0.0.1:52266, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T21:14:13,599 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42485-0x1015f3f4af00000, quorum=127.0.0.1:52266, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/849dee24a242,42485,1732223652547 2024-11-21T21:14:13,628 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41213-0x1015f3f4af00001, quorum=127.0.0.1:52266, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-21T21:14:13,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42485-0x1015f3f4af00000, quorum=127.0.0.1:52266, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:14:13,629 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41213-0x1015f3f4af00001, quorum=127.0.0.1:52266, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:14:13,630 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42485-0x1015f3f4af00000, quorum=127.0.0.1:52266, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-21T21:14:13,631 INFO [master/849dee24a242:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/849dee24a242,42485,1732223652547 from backup master directory 2024-11-21T21:14:13,645 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41213-0x1015f3f4af00001, quorum=127.0.0.1:52266, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T21:14:13,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42485-0x1015f3f4af00000, quorum=127.0.0.1:52266, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/849dee24a242,42485,1732223652547 2024-11-21T21:14:13,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42485-0x1015f3f4af00000, quorum=127.0.0.1:52266, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T21:14:13,646 WARN [master/849dee24a242:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T21:14:13,646 INFO [master/849dee24a242:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=849dee24a242,42485,1732223652547 2024-11-21T21:14:13,649 INFO [master/849dee24a242:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-21T21:14:13,652 INFO [master/849dee24a242:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-21T21:14:13,720 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/hbase.id] with ID: 237381a1-2648-400b-a825-c26d79780e33 2024-11-21T21:14:13,721 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/.tmp/hbase.id 2024-11-21T21:14:13,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741826_1002 (size=42) 2024-11-21T21:14:13,734 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/.tmp/hbase.id]:[hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/hbase.id] 2024-11-21T21:14:13,788 INFO [master/849dee24a242:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T21:14:13,792 INFO [master/849dee24a242:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-21T21:14:13,810 INFO [master/849dee24a242:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 15ms. 2024-11-21T21:14:13,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42485-0x1015f3f4af00000, quorum=127.0.0.1:52266, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:14:13,868 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41213-0x1015f3f4af00001, quorum=127.0.0.1:52266, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:14:13,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741827_1003 (size=196) 2024-11-21T21:14:13,905 INFO [master/849dee24a242:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T21:14:13,908 INFO [master/849dee24a242:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-21T21:14:13,922 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:13,926 INFO [master/849dee24a242:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T21:14:13,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741828_1004 (size=1189) 2024-11-21T21:14:14,377 INFO [master/849dee24a242:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/MasterData/data/master/store 2024-11-21T21:14:14,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741829_1005 (size=34) 2024-11-21T21:14:14,799 INFO [master/849dee24a242:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-21T21:14:14,802 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:14:14,803 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-21T21:14:14,803 INFO [master/849dee24a242:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T21:14:14,803 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T21:14:14,805 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-21T21:14:14,805 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T21:14:14,806 INFO [master/849dee24a242:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T21:14:14,807 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732223654803Disabling compacts and flushes for region at 1732223654803Disabling writes for close at 1732223654805 (+2 ms)Writing region close event to WAL at 1732223654805Closed at 1732223654805 2024-11-21T21:14:14,810 WARN [master/849dee24a242:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/MasterData/data/master/store/.initializing 2024-11-21T21:14:14,810 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/MasterData/WALs/849dee24a242,42485,1732223652547 2024-11-21T21:14:14,820 INFO [master/849dee24a242:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T21:14:14,837 INFO [master/849dee24a242:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=849dee24a242%2C42485%2C1732223652547, suffix=, logDir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/MasterData/WALs/849dee24a242,42485,1732223652547, archiveDir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/MasterData/oldWALs, maxLogs=10 2024-11-21T21:14:14,859 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/MasterData/WALs/849dee24a242,42485,1732223652547/849dee24a242%2C42485%2C1732223652547.1732223654842, exclude list is [], retry=0 2024-11-21T21:14:14,874 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35127,DS-91989f13-82a6-43fa-8cfb-bce78569bdbd,DISK] 2024-11-21T21:14:14,877 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-21T21:14:14,908 INFO [master/849dee24a242:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/MasterData/WALs/849dee24a242,42485,1732223652547/849dee24a242%2C42485%2C1732223652547.1732223654842 2024-11-21T21:14:14,909 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36345:36345)] 2024-11-21T21:14:14,910 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-21T21:14:14,910 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:14:14,913 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T21:14:14,914 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T21:14:14,950 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T21:14:14,976 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-21T21:14:14,979 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:14,981 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T21:14:14,982 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T21:14:14,985 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-21T21:14:14,985 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:14,986 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:14:14,987 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T21:14:14,996 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-21T21:14:14,996 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:14,997 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:14:14,997 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T21:14:15,000 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-21T21:14:15,000 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:15,001 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:14:15,002 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T21:14:15,006 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-21T21:14:15,007 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-21T21:14:15,013 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T21:14:15,014 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T21:14:15,018 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T21:14:15,022 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T21:14:15,027 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T21:14:15,029 INFO [master/849dee24a242:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74706057, jitterRate=0.11320699751377106}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T21:14:15,035 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732223654925Initializing all the Stores at 1732223654927 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732223654928 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732223654929 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732223654929Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732223654929Cleaning up temporary data from old regions at 1732223655014 (+85 ms)Region opened successfully at 1732223655035 (+21 ms) 2024-11-21T21:14:15,036 INFO [master/849dee24a242:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-21T21:14:15,068 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21f84067, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=849dee24a242/172.17.0.2:0 2024-11-21T21:14:15,094 INFO [master/849dee24a242:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-21T21:14:15,103 INFO [master/849dee24a242:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-21T21:14:15,103 INFO [master/849dee24a242:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-21T21:14:15,105 INFO [master/849dee24a242:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T21:14:15,106 INFO [master/849dee24a242:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-21T21:14:15,110 INFO [master/849dee24a242:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-21T21:14:15,111 INFO [master/849dee24a242:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-21T21:14:15,134 INFO [master/849dee24a242:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-21T21:14:15,145 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42485-0x1015f3f4af00000, quorum=127.0.0.1:52266, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-21T21:14:15,315 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-21T21:14:15,321 INFO [master/849dee24a242:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-21T21:14:15,323 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42485-0x1015f3f4af00000, quorum=127.0.0.1:52266, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-21T21:14:15,390 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-21T21:14:15,396 INFO [master/849dee24a242:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-21T21:14:15,402 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42485-0x1015f3f4af00000, quorum=127.0.0.1:52266, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-21T21:14:15,457 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-21T21:14:15,460 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42485-0x1015f3f4af00000, quorum=127.0.0.1:52266, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-21T21:14:15,512 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-21T21:14:15,531 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42485-0x1015f3f4af00000, quorum=127.0.0.1:52266, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-21T21:14:15,589 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-21T21:14:15,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42485-0x1015f3f4af00000, quorum=127.0.0.1:52266, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T21:14:15,686 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41213-0x1015f3f4af00001, quorum=127.0.0.1:52266, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T21:14:15,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42485-0x1015f3f4af00000, quorum=127.0.0.1:52266, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:14:15,687 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41213-0x1015f3f4af00001, quorum=127.0.0.1:52266, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:14:15,692 INFO [master/849dee24a242:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=849dee24a242,42485,1732223652547, sessionid=0x1015f3f4af00000, setting cluster-up flag (Was=false) 2024-11-21T21:14:15,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42485-0x1015f3f4af00000, quorum=127.0.0.1:52266, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:14:15,854 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41213-0x1015f3f4af00001, quorum=127.0.0.1:52266, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:14:16,066 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-21T21:14:16,072 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=849dee24a242,42485,1732223652547 2024-11-21T21:14:16,193 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41213-0x1015f3f4af00001, quorum=127.0.0.1:52266, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:14:16,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42485-0x1015f3f4af00000, quorum=127.0.0.1:52266, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:14:16,404 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-21T21:14:16,408 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=849dee24a242,42485,1732223652547 2024-11-21T21:14:16,417 INFO [master/849dee24a242:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-21T21:14:16,472 INFO [RS:0;849dee24a242:41213 {}] regionserver.HRegionServer(746): ClusterId : 237381a1-2648-400b-a825-c26d79780e33 2024-11-21T21:14:16,474 DEBUG [RS:0;849dee24a242:41213 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-21T21:14:16,490 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T21:14:16,500 INFO [master/849dee24a242:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-21T21:14:16,506 INFO [master/849dee24a242:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-21T21:14:16,510 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 849dee24a242,42485,1732223652547 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-21T21:14:16,557 DEBUG [RS:0;849dee24a242:41213 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-21T21:14:16,558 DEBUG [RS:0;849dee24a242:41213 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-21T21:14:16,558 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/849dee24a242:0, corePoolSize=5, maxPoolSize=5 2024-11-21T21:14:16,558 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/849dee24a242:0, corePoolSize=5, maxPoolSize=5 2024-11-21T21:14:16,559 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/849dee24a242:0, corePoolSize=5, maxPoolSize=5 2024-11-21T21:14:16,559 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/849dee24a242:0, corePoolSize=5, maxPoolSize=5 2024-11-21T21:14:16,559 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/849dee24a242:0, corePoolSize=10, maxPoolSize=10 2024-11-21T21:14:16,560 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/849dee24a242:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:14:16,560 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/849dee24a242:0, corePoolSize=2, maxPoolSize=2 2024-11-21T21:14:16,560 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/849dee24a242:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:14:16,562 INFO [master/849dee24a242:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732223686562 2024-11-21T21:14:16,564 INFO [master/849dee24a242:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-21T21:14:16,565 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T21:14:16,565 INFO [master/849dee24a242:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-21T21:14:16,566 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-21T21:14:16,570 INFO [master/849dee24a242:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-21T21:14:16,571 INFO [master/849dee24a242:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-21T21:14:16,571 INFO [master/849dee24a242:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-21T21:14:16,571 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:16,571 INFO [master/849dee24a242:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-21T21:14:16,571 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-21T21:14:16,573 INFO [master/849dee24a242:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-21T21:14:16,577 INFO [master/849dee24a242:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-21T21:14:16,579 INFO [master/849dee24a242:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-21T21:14:16,579 INFO [master/849dee24a242:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-21T21:14:16,588 INFO [master/849dee24a242:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-21T21:14:16,588 INFO [master/849dee24a242:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-21T21:14:16,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741831_1007 (size=1321) 2024-11-21T21:14:16,590 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/849dee24a242:0:becomeActiveMaster-HFileCleaner.large.0-1732223656590,5,FailOnTimeoutGroup] 2024-11-21T21:14:16,591 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/849dee24a242:0:becomeActiveMaster-HFileCleaner.small.0-1732223656590,5,FailOnTimeoutGroup] 2024-11-21T21:14:16,591 INFO [master/849dee24a242:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-21T21:14:16,591 INFO [master/849dee24a242:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-21T21:14:16,592 INFO [master/849dee24a242:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-21T21:14:16,592 INFO [master/849dee24a242:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-21T21:14:16,647 DEBUG [RS:0;849dee24a242:41213 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-21T21:14:16,648 DEBUG [RS:0;849dee24a242:41213 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32ababd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=849dee24a242/172.17.0.2:0 2024-11-21T21:14:16,669 DEBUG [RS:0;849dee24a242:41213 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;849dee24a242:41213 2024-11-21T21:14:16,673 INFO [RS:0;849dee24a242:41213 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-21T21:14:16,673 INFO [RS:0;849dee24a242:41213 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-21T21:14:16,674 DEBUG [RS:0;849dee24a242:41213 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-21T21:14:16,677 INFO [RS:0;849dee24a242:41213 {}] regionserver.HRegionServer(2659): reportForDuty to master=849dee24a242,42485,1732223652547 with port=41213, startcode=1732223653469 2024-11-21T21:14:16,690 DEBUG [RS:0;849dee24a242:41213 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-21T21:14:16,725 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51083, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-21T21:14:16,730 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42485 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 849dee24a242,41213,1732223653469 2024-11-21T21:14:16,733 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42485 {}] master.ServerManager(517): Registering regionserver=849dee24a242,41213,1732223653469 2024-11-21T21:14:16,745 DEBUG [RS:0;849dee24a242:41213 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f 2024-11-21T21:14:16,745 DEBUG [RS:0;849dee24a242:41213 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43233 2024-11-21T21:14:16,745 DEBUG [RS:0;849dee24a242:41213 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-21T21:14:16,788 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42485-0x1015f3f4af00000, quorum=127.0.0.1:52266, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T21:14:16,789 DEBUG [RS:0;849dee24a242:41213 {}] zookeeper.ZKUtil(111): regionserver:41213-0x1015f3f4af00001, quorum=127.0.0.1:52266, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/849dee24a242,41213,1732223653469 2024-11-21T21:14:16,789 WARN [RS:0;849dee24a242:41213 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T21:14:16,789 INFO [RS:0;849dee24a242:41213 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T21:14:16,790 DEBUG [RS:0;849dee24a242:41213 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/WALs/849dee24a242,41213,1732223653469 2024-11-21T21:14:16,792 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [849dee24a242,41213,1732223653469] 2024-11-21T21:14:16,820 INFO [RS:0;849dee24a242:41213 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-21T21:14:16,836 INFO [RS:0;849dee24a242:41213 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-21T21:14:16,841 INFO [RS:0;849dee24a242:41213 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T21:14:16,841 INFO [RS:0;849dee24a242:41213 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T21:14:16,842 INFO [RS:0;849dee24a242:41213 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-21T21:14:16,847 INFO [RS:0;849dee24a242:41213 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-21T21:14:16,848 INFO [RS:0;849dee24a242:41213 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-21T21:14:16,849 DEBUG [RS:0;849dee24a242:41213 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/849dee24a242:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:14:16,849 DEBUG [RS:0;849dee24a242:41213 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/849dee24a242:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:14:16,849 DEBUG [RS:0;849dee24a242:41213 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/849dee24a242:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:14:16,850 DEBUG [RS:0;849dee24a242:41213 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/849dee24a242:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:14:16,850 DEBUG [RS:0;849dee24a242:41213 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/849dee24a242:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:14:16,850 DEBUG [RS:0;849dee24a242:41213 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/849dee24a242:0, corePoolSize=2, maxPoolSize=2 2024-11-21T21:14:16,850 DEBUG [RS:0;849dee24a242:41213 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:14:16,850 DEBUG [RS:0;849dee24a242:41213 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/849dee24a242:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:14:16,851 DEBUG [RS:0;849dee24a242:41213 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/849dee24a242:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:14:16,851 DEBUG [RS:0;849dee24a242:41213 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/849dee24a242:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:14:16,851 DEBUG [RS:0;849dee24a242:41213 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/849dee24a242:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:14:16,851 DEBUG [RS:0;849dee24a242:41213 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/849dee24a242:0, corePoolSize=1, maxPoolSize=1 2024-11-21T21:14:16,851 DEBUG [RS:0;849dee24a242:41213 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/849dee24a242:0, corePoolSize=3, maxPoolSize=3 2024-11-21T21:14:16,851 DEBUG [RS:0;849dee24a242:41213 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0, corePoolSize=3, maxPoolSize=3 2024-11-21T21:14:16,852 INFO [RS:0;849dee24a242:41213 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T21:14:16,852 INFO [RS:0;849dee24a242:41213 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T21:14:16,853 INFO [RS:0;849dee24a242:41213 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T21:14:16,853 INFO [RS:0;849dee24a242:41213 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-21T21:14:16,853 INFO [RS:0;849dee24a242:41213 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-21T21:14:16,853 INFO [RS:0;849dee24a242:41213 {}] hbase.ChoreService(168): Chore ScheduledChore name=849dee24a242,41213,1732223653469-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T21:14:16,872 INFO [RS:0;849dee24a242:41213 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-21T21:14:16,875 INFO [RS:0;849dee24a242:41213 {}] hbase.ChoreService(168): Chore ScheduledChore name=849dee24a242,41213,1732223653469-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T21:14:16,875 INFO [RS:0;849dee24a242:41213 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T21:14:16,875 INFO [RS:0;849dee24a242:41213 {}] regionserver.Replication(171): 849dee24a242,41213,1732223653469 started 2024-11-21T21:14:16,893 INFO [RS:0;849dee24a242:41213 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T21:14:16,894 INFO [RS:0;849dee24a242:41213 {}] regionserver.HRegionServer(1482): Serving as 849dee24a242,41213,1732223653469, RpcServer on 849dee24a242/172.17.0.2:41213, sessionid=0x1015f3f4af00001 2024-11-21T21:14:16,894 DEBUG [RS:0;849dee24a242:41213 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-21T21:14:16,895 DEBUG [RS:0;849dee24a242:41213 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 849dee24a242,41213,1732223653469 2024-11-21T21:14:16,895 DEBUG [RS:0;849dee24a242:41213 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '849dee24a242,41213,1732223653469' 2024-11-21T21:14:16,895 DEBUG [RS:0;849dee24a242:41213 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-21T21:14:16,896 DEBUG [RS:0;849dee24a242:41213 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-21T21:14:16,897 DEBUG [RS:0;849dee24a242:41213 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-21T21:14:16,897 DEBUG [RS:0;849dee24a242:41213 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-21T21:14:16,897 DEBUG [RS:0;849dee24a242:41213 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 849dee24a242,41213,1732223653469 2024-11-21T21:14:16,897 DEBUG [RS:0;849dee24a242:41213 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '849dee24a242,41213,1732223653469' 2024-11-21T21:14:16,897 DEBUG [RS:0;849dee24a242:41213 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-21T21:14:16,898 DEBUG [RS:0;849dee24a242:41213 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-21T21:14:16,898 DEBUG [RS:0;849dee24a242:41213 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-21T21:14:16,899 INFO [RS:0;849dee24a242:41213 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-21T21:14:16,899 INFO [RS:0;849dee24a242:41213 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-21T21:14:16,993 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-21T21:14:16,993 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f 2024-11-21T21:14:17,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741832_1008 (size=32) 2024-11-21T21:14:17,005 INFO [RS:0;849dee24a242:41213 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T21:14:17,008 INFO [RS:0;849dee24a242:41213 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=849dee24a242%2C41213%2C1732223653469, suffix=, logDir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/WALs/849dee24a242,41213,1732223653469, archiveDir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/oldWALs, maxLogs=32 2024-11-21T21:14:17,025 DEBUG [RS:0;849dee24a242:41213 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/WALs/849dee24a242,41213,1732223653469/849dee24a242%2C41213%2C1732223653469.1732223657010, exclude list is [], retry=0 2024-11-21T21:14:17,030 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35127,DS-91989f13-82a6-43fa-8cfb-bce78569bdbd,DISK] 2024-11-21T21:14:17,034 INFO [RS:0;849dee24a242:41213 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/WALs/849dee24a242,41213,1732223653469/849dee24a242%2C41213%2C1732223653469.1732223657010 2024-11-21T21:14:17,035 DEBUG [RS:0;849dee24a242:41213 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36345:36345)] 2024-11-21T21:14:17,404 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:14:17,408 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-21T21:14:17,411 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T21:14:17,411 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:17,413 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T21:14:17,413 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-21T21:14:17,417 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T21:14:17,417 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:17,419 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T21:14:17,419 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-21T21:14:17,423 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T21:14:17,423 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:17,425 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T21:14:17,425 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-21T21:14:17,428 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T21:14:17,428 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:17,429 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T21:14:17,430 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T21:14:17,431 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/hbase/meta/1588230740 2024-11-21T21:14:17,432 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/hbase/meta/1588230740 2024-11-21T21:14:17,437 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T21:14:17,437 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T21:14:17,440 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T21:14:17,443 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T21:14:17,449 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T21:14:17,450 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67924172, jitterRate=0.012149035930633545}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T21:14:17,454 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732223657405Initializing all the Stores at 1732223657407 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732223657407Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732223657407Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732223657407Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732223657408 (+1 ms)Cleaning up temporary data from old regions at 1732223657437 (+29 ms)Region opened successfully at 1732223657454 (+17 ms) 2024-11-21T21:14:17,454 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T21:14:17,454 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T21:14:17,455 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T21:14:17,455 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T21:14:17,455 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T21:14:17,461 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T21:14:17,462 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732223657454Disabling compacts and flushes for region at 1732223657454Disabling writes for close at 1732223657455 (+1 ms)Writing region close event to WAL at 1732223657461 (+6 ms)Closed at 1732223657461 2024-11-21T21:14:17,468 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T21:14:17,469 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-21T21:14:17,478 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-21T21:14:17,488 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T21:14:17,492 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-21T21:14:17,644 DEBUG [849dee24a242:42485 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-21T21:14:17,655 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=849dee24a242,41213,1732223653469 2024-11-21T21:14:17,662 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 849dee24a242,41213,1732223653469, state=OPENING 2024-11-21T21:14:17,687 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-21T21:14:17,697 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41213-0x1015f3f4af00001, quorum=127.0.0.1:52266, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:14:17,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42485-0x1015f3f4af00000, quorum=127.0.0.1:52266, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:14:17,699 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T21:14:17,699 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T21:14:17,701 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T21:14:17,703 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=849dee24a242,41213,1732223653469}] 2024-11-21T21:14:17,883 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-21T21:14:17,886 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49445, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-21T21:14:17,897 INFO [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-21T21:14:17,897 INFO [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T21:14:17,898 INFO [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-21T21:14:17,901 INFO [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=849dee24a242%2C41213%2C1732223653469.meta, suffix=.meta, logDir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/WALs/849dee24a242,41213,1732223653469, archiveDir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/oldWALs, maxLogs=32 2024-11-21T21:14:17,915 DEBUG [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/WALs/849dee24a242,41213,1732223653469/849dee24a242%2C41213%2C1732223653469.meta.1732223657903.meta, exclude list is [], retry=0 2024-11-21T21:14:17,918 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35127,DS-91989f13-82a6-43fa-8cfb-bce78569bdbd,DISK] 2024-11-21T21:14:17,922 INFO [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/WALs/849dee24a242,41213,1732223653469/849dee24a242%2C41213%2C1732223653469.meta.1732223657903.meta 2024-11-21T21:14:17,922 DEBUG [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36345:36345)] 2024-11-21T21:14:17,922 DEBUG [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-21T21:14:17,924 DEBUG [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-21T21:14:17,926 DEBUG [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-21T21:14:17,930 INFO [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-21T21:14:17,934 DEBUG [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-21T21:14:17,935 DEBUG [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:14:17,935 DEBUG [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-21T21:14:17,935 DEBUG [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-21T21:14:17,938 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-21T21:14:17,941 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T21:14:17,941 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:17,942 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T21:14:17,942 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-21T21:14:17,944 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T21:14:17,944 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:17,945 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T21:14:17,945 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-21T21:14:17,947 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T21:14:17,948 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:17,948 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T21:14:17,949 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-21T21:14:17,951 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T21:14:17,951 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:17,952 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T21:14:17,952 DEBUG [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T21:14:17,954 DEBUG [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/hbase/meta/1588230740 2024-11-21T21:14:17,957 DEBUG [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/hbase/meta/1588230740 2024-11-21T21:14:17,960 DEBUG [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T21:14:17,960 DEBUG [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T21:14:17,961 DEBUG [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T21:14:17,965 DEBUG [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T21:14:17,966 INFO [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75107540, jitterRate=0.1191895604133606}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T21:14:17,967 DEBUG [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-21T21:14:17,969 DEBUG [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732223657936Writing region info on filesystem at 1732223657936Initializing all the Stores at 1732223657938 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732223657938Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732223657938Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732223657938Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732223657938Cleaning up temporary data from old regions at 1732223657960 (+22 ms)Running coprocessor post-open hooks at 1732223657967 (+7 ms)Region opened successfully at 1732223657969 (+2 ms) 2024-11-21T21:14:17,975 INFO [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732223657873 2024-11-21T21:14:17,988 DEBUG [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-21T21:14:17,988 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=849dee24a242,41213,1732223653469 2024-11-21T21:14:17,988 INFO [RS_OPEN_META-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-21T21:14:17,992 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 849dee24a242,41213,1732223653469, state=OPEN 2024-11-21T21:14:18,029 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41213-0x1015f3f4af00001, quorum=127.0.0.1:52266, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T21:14:18,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42485-0x1015f3f4af00000, quorum=127.0.0.1:52266, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T21:14:18,029 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T21:14:18,029 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T21:14:18,030 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:18,038 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-21T21:14:18,039 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=849dee24a242,41213,1732223653469 in 328 msec 2024-11-21T21:14:18,047 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-21T21:14:18,047 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 564 msec 2024-11-21T21:14:18,048 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T21:14:18,048 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-21T21:14:18,069 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:14:18,070 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:14:18,087 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:14:18,089 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40593, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:14:18,114 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6660 sec 2024-11-21T21:14:18,115 INFO [master/849dee24a242:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732223658114, completionTime=-1 2024-11-21T21:14:18,117 INFO [master/849dee24a242:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-21T21:14:18,117 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T21:14:18,138 INFO [master/849dee24a242:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-21T21:14:18,139 INFO [master/849dee24a242:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732223718139 2024-11-21T21:14:18,139 INFO [master/849dee24a242:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732223778139 2024-11-21T21:14:18,139 INFO [master/849dee24a242:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 21 msec 2024-11-21T21:14:18,141 INFO [master/849dee24a242:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=849dee24a242,42485,1732223652547-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T21:14:18,142 INFO [master/849dee24a242:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=849dee24a242,42485,1732223652547-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T21:14:18,142 INFO [master/849dee24a242:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=849dee24a242,42485,1732223652547-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T21:14:18,144 INFO [master/849dee24a242:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-849dee24a242:42485, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T21:14:18,145 INFO [master/849dee24a242:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-21T21:14:18,146 INFO [master/849dee24a242:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-21T21:14:18,152 DEBUG [master/849dee24a242:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-21T21:14:18,174 INFO [master/849dee24a242:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.528sec 2024-11-21T21:14:18,175 INFO [master/849dee24a242:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-21T21:14:18,176 INFO [master/849dee24a242:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-21T21:14:18,177 INFO [master/849dee24a242:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-21T21:14:18,178 INFO [master/849dee24a242:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-21T21:14:18,178 INFO [master/849dee24a242:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-21T21:14:18,178 INFO [master/849dee24a242:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=849dee24a242,42485,1732223652547-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T21:14:18,179 INFO [master/849dee24a242:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=849dee24a242,42485,1732223652547-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-21T21:14:18,187 DEBUG [master/849dee24a242:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-21T21:14:18,188 INFO [master/849dee24a242:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-21T21:14:18,188 INFO [master/849dee24a242:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=849dee24a242,42485,1732223652547-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T21:14:18,283 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bca31d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:18,285 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-21T21:14:18,285 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-21T21:14:18,287 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:14:18,289 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:14:18,296 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:14:18,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:14:18,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:14:18,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28364ad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:18,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:14:18,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:14:18,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:14:18,304 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50884, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:14:18,306 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@512c5941, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:18,307 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:14:18,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:14:18,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:14:18,318 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39484, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:14:18,338 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=849dee24a242,42485,1732223652547 2024-11-21T21:14:18,351 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=304, ProcessCount=11, AvailableMemoryMB=12180 2024-11-21T21:14:18,359 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T21:14:18,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 849dee24a242,42485,1732223652547 2024-11-21T21:14:18,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@61e4ff20 2024-11-21T21:14:18,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T21:14:18,392 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50894, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T21:14:18,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T21:14:18,398 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T21:14:18,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T21:14:18,412 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T21:14:18,414 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-21T21:14:18,414 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:18,418 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T21:14:18,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T21:14:18,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741835_1011 (size=1061) 2024-11-21T21:14:18,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T21:14:18,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T21:14:18,841 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f 2024-11-21T21:14:18,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741836_1012 (size=53) 2024-11-21T21:14:19,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T21:14:19,254 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:14:19,254 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ae951cc3e18af3209220d547b1b63947, disabling compactions & flushes 2024-11-21T21:14:19,254 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:19,255 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:19,255 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. after waiting 0 ms 2024-11-21T21:14:19,255 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:19,255 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:19,256 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ae951cc3e18af3209220d547b1b63947: Waiting for close lock at 1732223659254Disabling compacts and flushes for region at 1732223659254Disabling writes for close at 1732223659255 (+1 ms)Writing region close event to WAL at 1732223659255Closed at 1732223659255 2024-11-21T21:14:19,260 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T21:14:19,267 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732223659260"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732223659260"}]},"ts":"1732223659260"} 2024-11-21T21:14:19,272 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T21:14:19,274 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T21:14:19,277 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732223659274"}]},"ts":"1732223659274"} 2024-11-21T21:14:19,281 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T21:14:19,282 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae951cc3e18af3209220d547b1b63947, ASSIGN}] 2024-11-21T21:14:19,284 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae951cc3e18af3209220d547b1b63947, ASSIGN 2024-11-21T21:14:19,286 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae951cc3e18af3209220d547b1b63947, ASSIGN; state=OFFLINE, location=849dee24a242,41213,1732223653469; forceNewPlan=false, retain=false 2024-11-21T21:14:19,439 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ae951cc3e18af3209220d547b1b63947, regionState=OPENING, regionLocation=849dee24a242,41213,1732223653469 2024-11-21T21:14:19,448 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae951cc3e18af3209220d547b1b63947, ASSIGN because future has completed 2024-11-21T21:14:19,449 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469}] 2024-11-21T21:14:19,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T21:14:19,611 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:19,611 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} 2024-11-21T21:14:19,612 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ae951cc3e18af3209220d547b1b63947 2024-11-21T21:14:19,612 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:14:19,612 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for ae951cc3e18af3209220d547b1b63947 2024-11-21T21:14:19,612 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for ae951cc3e18af3209220d547b1b63947 2024-11-21T21:14:19,615 INFO [StoreOpener-ae951cc3e18af3209220d547b1b63947-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ae951cc3e18af3209220d547b1b63947 2024-11-21T21:14:19,619 INFO [StoreOpener-ae951cc3e18af3209220d547b1b63947-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T21:14:19,620 INFO [StoreOpener-ae951cc3e18af3209220d547b1b63947-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae951cc3e18af3209220d547b1b63947 columnFamilyName A 2024-11-21T21:14:19,620 DEBUG [StoreOpener-ae951cc3e18af3209220d547b1b63947-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:19,621 INFO [StoreOpener-ae951cc3e18af3209220d547b1b63947-1 {}] regionserver.HStore(327): Store=ae951cc3e18af3209220d547b1b63947/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:14:19,621 INFO [StoreOpener-ae951cc3e18af3209220d547b1b63947-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ae951cc3e18af3209220d547b1b63947 2024-11-21T21:14:19,624 INFO [StoreOpener-ae951cc3e18af3209220d547b1b63947-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T21:14:19,624 INFO [StoreOpener-ae951cc3e18af3209220d547b1b63947-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae951cc3e18af3209220d547b1b63947 columnFamilyName B 2024-11-21T21:14:19,625 DEBUG [StoreOpener-ae951cc3e18af3209220d547b1b63947-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:19,625 INFO [StoreOpener-ae951cc3e18af3209220d547b1b63947-1 {}] regionserver.HStore(327): Store=ae951cc3e18af3209220d547b1b63947/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:14:19,626 INFO [StoreOpener-ae951cc3e18af3209220d547b1b63947-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ae951cc3e18af3209220d547b1b63947 2024-11-21T21:14:19,628 INFO [StoreOpener-ae951cc3e18af3209220d547b1b63947-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T21:14:19,628 INFO [StoreOpener-ae951cc3e18af3209220d547b1b63947-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae951cc3e18af3209220d547b1b63947 columnFamilyName C 2024-11-21T21:14:19,629 DEBUG [StoreOpener-ae951cc3e18af3209220d547b1b63947-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:19,630 INFO [StoreOpener-ae951cc3e18af3209220d547b1b63947-1 {}] regionserver.HStore(327): Store=ae951cc3e18af3209220d547b1b63947/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:14:19,630 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:19,630 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for ae951cc3e18af3209220d547b1b63947 2024-11-21T21:14:19,632 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947 2024-11-21T21:14:19,633 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947 2024-11-21T21:14:19,635 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for ae951cc3e18af3209220d547b1b63947 2024-11-21T21:14:19,635 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for ae951cc3e18af3209220d547b1b63947 2024-11-21T21:14:19,636 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T21:14:19,639 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for ae951cc3e18af3209220d547b1b63947 2024-11-21T21:14:19,643 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T21:14:19,644 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened ae951cc3e18af3209220d547b1b63947; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73911676, jitterRate=0.10136979818344116}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T21:14:19,644 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ae951cc3e18af3209220d547b1b63947 2024-11-21T21:14:19,646 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for ae951cc3e18af3209220d547b1b63947: Running coprocessor pre-open hook at 1732223659612Writing region info on filesystem at 1732223659612Initializing all the Stores at 1732223659615 (+3 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732223659615Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732223659615Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732223659615Cleaning up temporary data from old regions at 1732223659635 (+20 ms)Running coprocessor post-open hooks at 1732223659644 (+9 ms)Region opened successfully at 1732223659645 (+1 ms) 2024-11-21T21:14:19,648 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., pid=6, masterSystemTime=1732223659604 2024-11-21T21:14:19,651 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:19,652 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:19,653 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ae951cc3e18af3209220d547b1b63947, regionState=OPEN, openSeqNum=2, regionLocation=849dee24a242,41213,1732223653469 2024-11-21T21:14:19,657 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 because future has completed 2024-11-21T21:14:19,664 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-21T21:14:19,664 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 in 211 msec 2024-11-21T21:14:19,669 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-21T21:14:19,670 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae951cc3e18af3209220d547b1b63947, ASSIGN in 382 msec 2024-11-21T21:14:19,671 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T21:14:19,671 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732223659671"}]},"ts":"1732223659671"} 2024-11-21T21:14:19,675 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T21:14:19,677 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T21:14:19,682 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2760 sec 2024-11-21T21:14:20,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T21:14:20,583 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:14:20,588 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f0b3b59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:20,588 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:14:20,588 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:14:20,590 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:14:20,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:14:20,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:14:20,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a220135, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:20,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:14:20,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:14:20,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:14:20,593 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50918, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:14:20,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@322a9c3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:20,595 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:14:20,596 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:14:20,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:14:20,599 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39496, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:14:20,601 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T21:14:20,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 849dee24a242,42485,1732223652547 2024-11-21T21:14:20,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@63481454 2024-11-21T21:14:20,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T21:14:20,606 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50924, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T21:14:20,610 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@340b46e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:20,611 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:14:20,611 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:14:20,613 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:14:20,613 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:14:20,613 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:14:20,614 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ccb4eda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:20,614 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:14:20,614 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:14:20,615 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:14:20,616 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50936, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:14:20,618 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fc8436a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:20,619 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6edf4fef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:20,619 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:14:20,619 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:14:20,620 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:14:20,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:14:20,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:14:20,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21d7efe6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:20,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:14:20,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:14:20,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:14:20,624 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50948, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:14:20,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7230d8e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:20,626 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21f4b80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:20,626 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:14:20,626 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:14:20,628 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:14:20,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:14:20,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:14:20,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64257f01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:20,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:14:20,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:14:20,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:14:20,632 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:14:20,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14ba4844, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:20,634 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52294ca5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:20,634 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:14:20,634 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:14:20,636 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:14:20,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:14:20,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:14:20,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55ae7626, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:20,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:14:20,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:14:20,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:14:20,639 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50986, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:14:20,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d73963e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:20,641 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d7926de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:20,641 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:14:20,641 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:14:20,642 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:14:20,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:14:20,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:14:20,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f7f2409, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:20,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:14:20,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:14:20,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:14:20,645 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51004, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:14:20,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4787ae6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:20,647 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b48425, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:20,647 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:14:20,648 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:14:20,649 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:14:20,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:14:20,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:14:20,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@eac2906, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:20,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:14:20,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:14:20,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:14:20,651 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:14:20,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15a3e301, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:20,653 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4454ce2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:20,653 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:14:20,653 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:14:20,654 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:14:20,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:14:20,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:14:20,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@788043de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:20,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:14:20,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:14:20,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:14:20,657 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51036, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:14:20,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@421973e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:20,659 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74a9c1c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:20,659 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:14:20,660 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:14:20,661 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:14:20,661 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:14:20,661 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:14:20,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fe80a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:20,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:14:20,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:14:20,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:14:20,663 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51052, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:14:20,664 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76c8af47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:20,666 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cadbd93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:20,666 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:14:20,667 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:14:20,668 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:14:20,668 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:14:20,668 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:14:20,668 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@676ef68d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:20,668 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:14:20,669 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:14:20,669 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:14:20,670 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51064, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:14:20,671 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5addfb03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:20,674 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:14:20,674 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:14:20,674 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:14:20,675 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:14:20,676 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:14:20,676 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:14:20,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:14:20,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:14:20,677 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:14:20,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:14:20,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:14:20,677 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:14:20,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:14:20,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:14:20,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:14:20,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:14:20,679 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39498, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:14:20,679 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39500, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:14:20,680 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39510, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:14:20,680 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39512, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:14:20,680 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39526, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:14:20,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:14:20,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:14:20,682 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:14:20,684 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39532, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:14:20,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2] 2024-11-21T21:14:20,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2] 2024-11-21T21:14:20,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2] 2024-11-21T21:14:20,690 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:14:20,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2] 2024-11-21T21:14:20,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2] 2024-11-21T21:14:20,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2] 2024-11-21T21:14:20,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-21T21:14:20,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T21:14:20,696 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:14:20,696 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:14:20,697 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:14:20,698 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:14:20,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:14:20,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:14:20,699 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39534, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:14:20,700 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:14:20,700 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39536, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:14:20,702 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:14:20,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2] 2024-11-21T21:14:20,704 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:14:20,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2] 2024-11-21T21:14:20,706 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:14:20,706 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:14:20,708 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39538, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:14:20,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2] 2024-11-21T21:14:20,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T21:14:20,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on ae951cc3e18af3209220d547b1b63947 2024-11-21T21:14:20,806 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae951cc3e18af3209220d547b1b63947 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T21:14:20,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=A 2024-11-21T21:14:20,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:20,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=B 2024-11-21T21:14:20,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:20,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=C 2024-11-21T21:14:20,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:20,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T21:14:20,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:20,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:20,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:20,883 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:20,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:20,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:20,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:20,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:20,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223720899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:20,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223720899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:20,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:20,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223720907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:20,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:20,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223720921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:20,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:20,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223720920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:20,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/6acc1e84ea42446cafd2f800780b19b4 is 50, key is test_row_0/A:col10/1732223660802/Put/seqid=0 2024-11-21T21:14:20,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:20,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:20,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:20,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:20,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:20,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:20,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:20,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:20,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:20,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:20,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:20,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:20,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:20,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:20,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:20,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741837_1013 (size=12001) 2024-11-21T21:14:20,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/6acc1e84ea42446cafd2f800780b19b4 2024-11-21T21:14:21,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T21:14:21,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T21:14:21,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:21,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:21,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:21,059 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:21,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:21,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:21,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:21,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223721073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:21,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:21,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:21,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:21,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:21,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:21,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223721074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:21,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:21,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223721074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:21,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:21,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:21,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223721075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:21,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:21,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:21,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:21,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223721073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:21,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:21,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:21,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:21,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:21,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:21,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:21,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:21,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:21,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/fe2594045ed34001a4bcc82eda2e7bc8 is 50, key is test_row_0/B:col10/1732223660802/Put/seqid=0 2024-11-21T21:14:21,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741838_1014 (size=12001) 2024-11-21T21:14:21,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T21:14:21,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:21,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:21,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:21,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:21,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:21,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:21,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:21,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223721281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:21,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:21,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:21,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:21,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:21,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223721291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:21,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:21,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:21,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:21,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:21,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223721291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:21,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:21,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:21,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:21,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:21,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223721292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:21,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:21,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223721293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:21,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:21,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:21,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:21,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:21,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:21,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:21,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T21:14:21,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T21:14:21,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:21,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:21,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:21,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:21,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:21,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:21,513 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/fe2594045ed34001a4bcc82eda2e7bc8 2024-11-21T21:14:21,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T21:14:21,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:21,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:21,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:21,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:21,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:21,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:21,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/d6bd91d28dfd4d15b9bfff7294f860d9 is 50, key is test_row_0/C:col10/1732223660802/Put/seqid=0 2024-11-21T21:14:21,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741839_1015 (size=12001) 2024-11-21T21:14:21,585 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/d6bd91d28dfd4d15b9bfff7294f860d9 2024-11-21T21:14:21,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:21,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223721601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:21,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/6acc1e84ea42446cafd2f800780b19b4 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/6acc1e84ea42446cafd2f800780b19b4 2024-11-21T21:14:21,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:21,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223721602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:21,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:21,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:21,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:21,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:21,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:21,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:21,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:21,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:21,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223721602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:21,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223721602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:21,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:21,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:21,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:21,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:21,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:21,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:21,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:21,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223721611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:21,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:21,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:21,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:21,626 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/6acc1e84ea42446cafd2f800780b19b4, entries=150, sequenceid=13, filesize=11.7 K 2024-11-21T21:14:21,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/fe2594045ed34001a4bcc82eda2e7bc8 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/fe2594045ed34001a4bcc82eda2e7bc8 2024-11-21T21:14:21,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/fe2594045ed34001a4bcc82eda2e7bc8, entries=150, sequenceid=13, filesize=11.7 K 2024-11-21T21:14:21,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/d6bd91d28dfd4d15b9bfff7294f860d9 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/d6bd91d28dfd4d15b9bfff7294f860d9 2024-11-21T21:14:21,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/d6bd91d28dfd4d15b9bfff7294f860d9, entries=150, sequenceid=13, filesize=11.7 K 2024-11-21T21:14:21,677 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for ae951cc3e18af3209220d547b1b63947 in 870ms, sequenceid=13, compaction requested=false 2024-11-21T21:14:21,678 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-21T21:14:21,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:21,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T21:14:21,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:21,688 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing ae951cc3e18af3209220d547b1b63947 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-21T21:14:21,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=A 2024-11-21T21:14:21,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:21,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=B 2024-11-21T21:14:21,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:21,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=C 2024-11-21T21:14:21,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:21,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/7b1d67a56ac34d93b9a119f8b3d3f8c4 is 50, key is test_row_0/A:col10/1732223660913/Put/seqid=0 2024-11-21T21:14:21,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741840_1016 (size=12001) 2024-11-21T21:14:21,721 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/7b1d67a56ac34d93b9a119f8b3d3f8c4 2024-11-21T21:14:21,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/d4bda57f98b24ff9b87d1e60f9053913 is 50, key is test_row_0/B:col10/1732223660913/Put/seqid=0 2024-11-21T21:14:21,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741841_1017 (size=12001) 2024-11-21T21:14:21,792 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/d4bda57f98b24ff9b87d1e60f9053913 2024-11-21T21:14:21,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/d13b2fa7021642cfbd52146b9a910f67 is 50, key is test_row_0/C:col10/1732223660913/Put/seqid=0 2024-11-21T21:14:21,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T21:14:21,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741842_1018 (size=12001) 2024-11-21T21:14:21,860 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/d13b2fa7021642cfbd52146b9a910f67 2024-11-21T21:14:21,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/7b1d67a56ac34d93b9a119f8b3d3f8c4 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/7b1d67a56ac34d93b9a119f8b3d3f8c4 2024-11-21T21:14:21,893 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/7b1d67a56ac34d93b9a119f8b3d3f8c4, entries=150, sequenceid=33, filesize=11.7 K 2024-11-21T21:14:21,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/d4bda57f98b24ff9b87d1e60f9053913 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/d4bda57f98b24ff9b87d1e60f9053913 2024-11-21T21:14:21,913 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/d4bda57f98b24ff9b87d1e60f9053913, entries=150, sequenceid=33, filesize=11.7 K 2024-11-21T21:14:21,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/d13b2fa7021642cfbd52146b9a910f67 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/d13b2fa7021642cfbd52146b9a910f67 2024-11-21T21:14:21,942 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/d13b2fa7021642cfbd52146b9a910f67, entries=150, sequenceid=33, filesize=11.7 K 2024-11-21T21:14:21,945 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for ae951cc3e18af3209220d547b1b63947 in 257ms, sequenceid=33, compaction requested=false 2024-11-21T21:14:21,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:21,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:21,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-21T21:14:21,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-21T21:14:21,952 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-21T21:14:21,952 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2460 sec 2024-11-21T21:14:21,957 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.2700 sec 2024-11-21T21:14:22,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on ae951cc3e18af3209220d547b1b63947 2024-11-21T21:14:22,152 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae951cc3e18af3209220d547b1b63947 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T21:14:22,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=A 2024-11-21T21:14:22,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:22,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=B 2024-11-21T21:14:22,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:22,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=C 2024-11-21T21:14:22,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:22,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/059758ef32d74cc7aa6f0403af5717a7 is 50, key is test_row_0/A:col10/1732223662151/Put/seqid=0 2024-11-21T21:14:22,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741843_1019 (size=14341) 2024-11-21T21:14:22,212 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/059758ef32d74cc7aa6f0403af5717a7 2024-11-21T21:14:22,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:22,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223722206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:22,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:22,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:22,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:22,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:22,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223722214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:22,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:22,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:22,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:22,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:22,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223722221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:22,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:22,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:22,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:22,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223722223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:22,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:22,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:22,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:22,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:22,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:22,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223722223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:22,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:22,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:22,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:22,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/cdbe0680a3714d6b9aa700a9f9fccfbd is 50, key is test_row_0/B:col10/1732223662151/Put/seqid=0 2024-11-21T21:14:22,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741844_1020 (size=12001) 2024-11-21T21:14:22,270 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/cdbe0680a3714d6b9aa700a9f9fccfbd 2024-11-21T21:14:22,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/e6f35a9c9d744f9899d4d17d2a4e6482 is 50, key is test_row_0/C:col10/1732223662151/Put/seqid=0 2024-11-21T21:14:22,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741845_1021 (size=12001) 2024-11-21T21:14:22,321 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/e6f35a9c9d744f9899d4d17d2a4e6482 2024-11-21T21:14:22,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:22,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223722330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:22,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:22,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:22,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:22,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/059758ef32d74cc7aa6f0403af5717a7 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/059758ef32d74cc7aa6f0403af5717a7 2024-11-21T21:14:22,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:22,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223722342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:22,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:22,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:22,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:22,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:22,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223722342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:22,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:22,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:22,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:22,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:22,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223722343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:22,349 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:22,349 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:22,349 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:22,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:22,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223722343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:22,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:22,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:22,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:22,355 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/059758ef32d74cc7aa6f0403af5717a7, entries=200, sequenceid=45, filesize=14.0 K 2024-11-21T21:14:22,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/cdbe0680a3714d6b9aa700a9f9fccfbd as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/cdbe0680a3714d6b9aa700a9f9fccfbd 2024-11-21T21:14:22,382 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/cdbe0680a3714d6b9aa700a9f9fccfbd, entries=150, sequenceid=45, filesize=11.7 K 2024-11-21T21:14:22,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/e6f35a9c9d744f9899d4d17d2a4e6482 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/e6f35a9c9d744f9899d4d17d2a4e6482 2024-11-21T21:14:22,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/e6f35a9c9d744f9899d4d17d2a4e6482, entries=150, sequenceid=45, filesize=11.7 K 2024-11-21T21:14:22,404 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for ae951cc3e18af3209220d547b1b63947 in 254ms, sequenceid=45, compaction requested=true 2024-11-21T21:14:22,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:22,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae951cc3e18af3209220d547b1b63947:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:14:22,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:22,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae951cc3e18af3209220d547b1b63947:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:14:22,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:22,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae951cc3e18af3209220d547b1b63947:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:14:22,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:14:22,411 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:14:22,413 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:14:22,417 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:14:22,419 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:14:22,419 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): ae951cc3e18af3209220d547b1b63947/A is initiating minor compaction (all files) 2024-11-21T21:14:22,419 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): ae951cc3e18af3209220d547b1b63947/B is initiating minor compaction (all files) 2024-11-21T21:14:22,420 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae951cc3e18af3209220d547b1b63947/B in TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:22,420 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/fe2594045ed34001a4bcc82eda2e7bc8, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/d4bda57f98b24ff9b87d1e60f9053913, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/cdbe0680a3714d6b9aa700a9f9fccfbd] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp, totalSize=35.2 K 2024-11-21T21:14:22,421 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae951cc3e18af3209220d547b1b63947/A in TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:22,421 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/6acc1e84ea42446cafd2f800780b19b4, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/7b1d67a56ac34d93b9a119f8b3d3f8c4, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/059758ef32d74cc7aa6f0403af5717a7] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp, totalSize=37.4 K 2024-11-21T21:14:22,422 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting fe2594045ed34001a4bcc82eda2e7bc8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732223660791 2024-11-21T21:14:22,423 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting d4bda57f98b24ff9b87d1e60f9053913, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732223660888 2024-11-21T21:14:22,423 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6acc1e84ea42446cafd2f800780b19b4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732223660791 2024-11-21T21:14:22,424 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting cdbe0680a3714d6b9aa700a9f9fccfbd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1732223662134 2024-11-21T21:14:22,424 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b1d67a56ac34d93b9a119f8b3d3f8c4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732223660888 2024-11-21T21:14:22,425 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 059758ef32d74cc7aa6f0403af5717a7, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1732223662134 2024-11-21T21:14:22,465 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae951cc3e18af3209220d547b1b63947#B#compaction#9 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:14:22,466 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/b37dcc48c61e4cc8a90322bfcbcee6a5 is 50, key is test_row_0/B:col10/1732223662151/Put/seqid=0 2024-11-21T21:14:22,468 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae951cc3e18af3209220d547b1b63947#A#compaction#10 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:14:22,469 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/903c4e60d57948db94a65a96331e9c7f is 50, key is test_row_0/A:col10/1732223662151/Put/seqid=0 2024-11-21T21:14:22,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741846_1022 (size=12104) 2024-11-21T21:14:22,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741847_1023 (size=12104) 2024-11-21T21:14:22,514 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/b37dcc48c61e4cc8a90322bfcbcee6a5 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/b37dcc48c61e4cc8a90322bfcbcee6a5 2024-11-21T21:14:22,518 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/903c4e60d57948db94a65a96331e9c7f as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/903c4e60d57948db94a65a96331e9c7f 2024-11-21T21:14:22,541 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae951cc3e18af3209220d547b1b63947/B of ae951cc3e18af3209220d547b1b63947 into b37dcc48c61e4cc8a90322bfcbcee6a5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:14:22,541 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:22,546 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae951cc3e18af3209220d547b1b63947/A of ae951cc3e18af3209220d547b1b63947 into 903c4e60d57948db94a65a96331e9c7f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:14:22,546 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:22,550 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., storeName=ae951cc3e18af3209220d547b1b63947/B, priority=13, startTime=1732223662409; duration=0sec 2024-11-21T21:14:22,551 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:14:22,551 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae951cc3e18af3209220d547b1b63947:B 2024-11-21T21:14:22,551 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:14:22,553 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., storeName=ae951cc3e18af3209220d547b1b63947/A, priority=13, startTime=1732223662406; duration=0sec 2024-11-21T21:14:22,554 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:22,554 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae951cc3e18af3209220d547b1b63947:A 2024-11-21T21:14:22,555 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:14:22,555 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): ae951cc3e18af3209220d547b1b63947/C is initiating minor compaction (all files) 2024-11-21T21:14:22,555 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae951cc3e18af3209220d547b1b63947/C in TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:22,555 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/d6bd91d28dfd4d15b9bfff7294f860d9, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/d13b2fa7021642cfbd52146b9a910f67, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/e6f35a9c9d744f9899d4d17d2a4e6482] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp, totalSize=35.2 K 2024-11-21T21:14:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on ae951cc3e18af3209220d547b1b63947 2024-11-21T21:14:22,557 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae951cc3e18af3209220d547b1b63947 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-21T21:14:22,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=A 2024-11-21T21:14:22,559 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting d6bd91d28dfd4d15b9bfff7294f860d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732223660791 2024-11-21T21:14:22,560 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting d13b2fa7021642cfbd52146b9a910f67, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732223660888 2024-11-21T21:14:22,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:14:22,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=B 2024-11-21T21:14:22,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:14:22,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=C 2024-11-21T21:14:22,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:14:22,564 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting e6f35a9c9d744f9899d4d17d2a4e6482, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1732223662134 2024-11-21T21:14:22,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/223636bcf5664d01b47f4ad6eec2f405 is 50, key is test_row_0/A:col10/1732223662208/Put/seqid=0 2024-11-21T21:14:22,596 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae951cc3e18af3209220d547b1b63947#C#compaction#12 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:14:22,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223722580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:22,597 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/0aeb8989d6cb4a84aa87d3a9e5248266 is 50, key is test_row_0/C:col10/1732223662151/Put/seqid=0 2024-11-21T21:14:22,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:22,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:22,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:22,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:22,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223722587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:22,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:22,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:22,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:22,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:22,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:22,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223722588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:22,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223722588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:22,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:22,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:22,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:22,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:22,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:22,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:22,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:22,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223722591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:22,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:22,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:22,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:22,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741849_1025 (size=12104) 2024-11-21T21:14:22,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741848_1024 (size=12001) 2024-11-21T21:14:22,630 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/223636bcf5664d01b47f4ad6eec2f405 2024-11-21T21:14:22,644 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/0aeb8989d6cb4a84aa87d3a9e5248266 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/0aeb8989d6cb4a84aa87d3a9e5248266 2024-11-21T21:14:22,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/2cb5b396ffda4221961aa2a226a9a390 is 50, key is test_row_0/B:col10/1732223662208/Put/seqid=0 2024-11-21T21:14:22,668 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae951cc3e18af3209220d547b1b63947/C of ae951cc3e18af3209220d547b1b63947 into 0aeb8989d6cb4a84aa87d3a9e5248266(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:14:22,669 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:22,669 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., storeName=ae951cc3e18af3209220d547b1b63947/C, priority=13, startTime=1732223662409; duration=0sec 2024-11-21T21:14:22,669 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:22,669 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae951cc3e18af3209220d547b1b63947:C 2024-11-21T21:14:22,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741850_1026 (size=12001) 2024-11-21T21:14:22,678 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/2cb5b396ffda4221961aa2a226a9a390 2024-11-21T21:14:22,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/e5da33c308dc4f48b12d2575f236cd0d is 50, key is test_row_0/C:col10/1732223662208/Put/seqid=0 2024-11-21T21:14:22,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:22,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223722704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:22,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:22,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:22,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:22,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:22,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223722711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:22,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:22,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:22,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:22,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:22,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223722711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:22,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:22,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:22,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:22,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:22,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223722711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:22,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:22,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:22,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:22,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741851_1027 (size=12001) 2024-11-21T21:14:22,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223722721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:22,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:22,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:22,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:22,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/e5da33c308dc4f48b12d2575f236cd0d 2024-11-21T21:14:22,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/223636bcf5664d01b47f4ad6eec2f405 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/223636bcf5664d01b47f4ad6eec2f405 2024-11-21T21:14:22,768 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/223636bcf5664d01b47f4ad6eec2f405, entries=150, sequenceid=68, filesize=11.7 K 2024-11-21T21:14:22,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/2cb5b396ffda4221961aa2a226a9a390 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/2cb5b396ffda4221961aa2a226a9a390 2024-11-21T21:14:22,797 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/2cb5b396ffda4221961aa2a226a9a390, entries=150, sequenceid=68, filesize=11.7 K 2024-11-21T21:14:22,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/e5da33c308dc4f48b12d2575f236cd0d as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/e5da33c308dc4f48b12d2575f236cd0d 2024-11-21T21:14:22,810 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/e5da33c308dc4f48b12d2575f236cd0d, entries=150, sequenceid=68, filesize=11.7 K 2024-11-21T21:14:22,812 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-21T21:14:22,815 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for ae951cc3e18af3209220d547b1b63947 in 258ms, sequenceid=68, compaction requested=false 2024-11-21T21:14:22,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:22,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T21:14:22,841 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:14:22,842 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:14:22,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-21T21:14:22,848 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:14:22,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T21:14:22,851 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:14:22,851 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:14:22,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on ae951cc3e18af3209220d547b1b63947 2024-11-21T21:14:22,913 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae951cc3e18af3209220d547b1b63947 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-21T21:14:22,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=A 2024-11-21T21:14:22,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:22,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=B 2024-11-21T21:14:22,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:22,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=C 2024-11-21T21:14:22,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:22,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/968a26425ab746069a9004f41a5719ea is 50, key is test_row_0/A:col10/1732223662586/Put/seqid=0 2024-11-21T21:14:22,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741852_1028 (size=12001) 2024-11-21T21:14:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T21:14:22,962 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/968a26425ab746069a9004f41a5719ea 2024-11-21T21:14:22,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223722974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:22,989 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/9a0a43faea614ee3a1005cf7dca35ae3 is 50, key is test_row_0/B:col10/1732223662586/Put/seqid=0 2024-11-21T21:14:22,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:22,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223722978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:22,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:22,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:22,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:22,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:22,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:22,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:22,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:22,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223722978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:22,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:22,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:22,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:22,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:22,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223722981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:22,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:22,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:22,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:22,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:22,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223722982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:22,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:22,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:22,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:23,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-21T21:14:23,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:23,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:23,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:23,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:23,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:23,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741853_1029 (size=12001) 2024-11-21T21:14:23,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:23,010 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/9a0a43faea614ee3a1005cf7dca35ae3 2024-11-21T21:14:23,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:23,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223723102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:23,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:23,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223723102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:23,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:23,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223723103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:23,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:23,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223723104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:23,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:23,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:23,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223723105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:23,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:23,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:23,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:23,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:23,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:23,115 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/fabf841369a14de0bfed325bd274ba07 is 50, key is test_row_0/C:col10/1732223662586/Put/seqid=0 2024-11-21T21:14:23,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741854_1030 (size=12001) 2024-11-21T21:14:23,133 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/fabf841369a14de0bfed325bd274ba07 2024-11-21T21:14:23,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/968a26425ab746069a9004f41a5719ea as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/968a26425ab746069a9004f41a5719ea 2024-11-21T21:14:23,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-21T21:14:23,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:23,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:23,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:23,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:23,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:23,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:23,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/968a26425ab746069a9004f41a5719ea, entries=150, sequenceid=81, filesize=11.7 K 2024-11-21T21:14:23,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T21:14:23,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/9a0a43faea614ee3a1005cf7dca35ae3 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/9a0a43faea614ee3a1005cf7dca35ae3 2024-11-21T21:14:23,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/9a0a43faea614ee3a1005cf7dca35ae3, entries=150, sequenceid=81, filesize=11.7 K 2024-11-21T21:14:23,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/fabf841369a14de0bfed325bd274ba07 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/fabf841369a14de0bfed325bd274ba07 2024-11-21T21:14:23,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/fabf841369a14de0bfed325bd274ba07, entries=150, sequenceid=81, filesize=11.7 K 2024-11-21T21:14:23,218 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for ae951cc3e18af3209220d547b1b63947 in 305ms, sequenceid=81, compaction requested=true 2024-11-21T21:14:23,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:23,219 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:14:23,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae951cc3e18af3209220d547b1b63947:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:14:23,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:23,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae951cc3e18af3209220d547b1b63947:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:14:23,220 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:14:23,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:23,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae951cc3e18af3209220d547b1b63947:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:14:23,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:14:23,222 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:14:23,223 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): ae951cc3e18af3209220d547b1b63947/A is initiating minor compaction (all files) 2024-11-21T21:14:23,223 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae951cc3e18af3209220d547b1b63947/A in TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:23,223 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/903c4e60d57948db94a65a96331e9c7f, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/223636bcf5664d01b47f4ad6eec2f405, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/968a26425ab746069a9004f41a5719ea] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp, totalSize=35.3 K 2024-11-21T21:14:23,224 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 903c4e60d57948db94a65a96331e9c7f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1732223662134 2024-11-21T21:14:23,225 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:14:23,225 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): ae951cc3e18af3209220d547b1b63947/B is initiating minor compaction (all files) 2024-11-21T21:14:23,226 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae951cc3e18af3209220d547b1b63947/B in TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:23,226 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/b37dcc48c61e4cc8a90322bfcbcee6a5, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/2cb5b396ffda4221961aa2a226a9a390, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/9a0a43faea614ee3a1005cf7dca35ae3] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp, totalSize=35.3 K 2024-11-21T21:14:23,226 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 223636bcf5664d01b47f4ad6eec2f405, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1732223662208 2024-11-21T21:14:23,229 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting b37dcc48c61e4cc8a90322bfcbcee6a5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1732223662134 2024-11-21T21:14:23,230 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 968a26425ab746069a9004f41a5719ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732223662582 2024-11-21T21:14:23,232 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cb5b396ffda4221961aa2a226a9a390, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1732223662208 2024-11-21T21:14:23,234 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a0a43faea614ee3a1005cf7dca35ae3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732223662582 2024-11-21T21:14:23,268 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae951cc3e18af3209220d547b1b63947#A#compaction#18 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:14:23,269 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/653f2f5b1c094038a95f14af19f28bb7 is 50, key is test_row_0/A:col10/1732223662586/Put/seqid=0 2024-11-21T21:14:23,287 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae951cc3e18af3209220d547b1b63947#B#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:14:23,288 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/3cedb04748894fd1bd65a4f4db283444 is 50, key is test_row_0/B:col10/1732223662586/Put/seqid=0 2024-11-21T21:14:23,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-21T21:14:23,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:23,318 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing ae951cc3e18af3209220d547b1b63947 3/3 column families, dataSize=114.05 KB heapSize=339.66 KB 2024-11-21T21:14:23,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741855_1031 (size=12207) 2024-11-21T21:14:23,321 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732223662972, max timestamp=1732223663314Num uniques 859; 2024-11-21T21:14:23,324 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T21:14:23,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on ae951cc3e18af3209220d547b1b63947 2024-11-21T21:14:23,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:23,328 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732223662972, max timestamp=1732223663314Num uniques 859; 2024-11-21T21:14:23,329 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T21:14:23,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=A 2024-11-21T21:14:23,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:14:23,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=B 2024-11-21T21:14:23,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:14:23,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=C 2024-11-21T21:14:23,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:14:23,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741856_1032 (size=12207) 2024-11-21T21:14:23,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/f0e1c06ad55b40a9860bb148af5a70d0 is 50, key is test_row_0/A:col10/1732223662977/Put/seqid=0 2024-11-21T21:14:23,371 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/3cedb04748894fd1bd65a4f4db283444 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/3cedb04748894fd1bd65a4f4db283444 2024-11-21T21:14:23,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741857_1033 (size=12001) 2024-11-21T21:14:23,377 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/f0e1c06ad55b40a9860bb148af5a70d0 2024-11-21T21:14:23,385 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae951cc3e18af3209220d547b1b63947/B of ae951cc3e18af3209220d547b1b63947 into 3cedb04748894fd1bd65a4f4db283444(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:14:23,386 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:23,387 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., storeName=ae951cc3e18af3209220d547b1b63947/B, priority=13, startTime=1732223663220; duration=0sec 2024-11-21T21:14:23,387 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:14:23,387 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae951cc3e18af3209220d547b1b63947:B 2024-11-21T21:14:23,388 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:14:23,390 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:14:23,390 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): ae951cc3e18af3209220d547b1b63947/C is initiating minor compaction (all files) 2024-11-21T21:14:23,391 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae951cc3e18af3209220d547b1b63947/C in TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:23,391 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/0aeb8989d6cb4a84aa87d3a9e5248266, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/e5da33c308dc4f48b12d2575f236cd0d, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/fabf841369a14de0bfed325bd274ba07] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp, totalSize=35.3 K 2024-11-21T21:14:23,392 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 0aeb8989d6cb4a84aa87d3a9e5248266, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1732223662134 2024-11-21T21:14:23,393 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting e5da33c308dc4f48b12d2575f236cd0d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1732223662208 2024-11-21T21:14:23,394 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting fabf841369a14de0bfed325bd274ba07, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732223662582 2024-11-21T21:14:23,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/17f81127c74048aba81497f87bd5e668 is 50, key is test_row_0/B:col10/1732223662977/Put/seqid=0 2024-11-21T21:14:23,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:23,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223723382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:23,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:23,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:23,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:23,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:23,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223723387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:23,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:23,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:23,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:23,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:23,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223723394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:23,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:23,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:23,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:23,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:23,415 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae951cc3e18af3209220d547b1b63947#C#compaction#22 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:14:23,416 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/f9090e80448d436ba2c6fd7b5683ab0e is 50, key is test_row_0/C:col10/1732223662586/Put/seqid=0 2024-11-21T21:14:23,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223723402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:23,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:23,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:23,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:23,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:23,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223723402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:23,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:23,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:23,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:23,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741858_1034 (size=12001) 2024-11-21T21:14:23,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741859_1035 (size=12207) 2024-11-21T21:14:23,451 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T21:14:23,455 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/f9090e80448d436ba2c6fd7b5683ab0e as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/f9090e80448d436ba2c6fd7b5683ab0e 2024-11-21T21:14:23,466 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae951cc3e18af3209220d547b1b63947/C of ae951cc3e18af3209220d547b1b63947 into f9090e80448d436ba2c6fd7b5683ab0e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:14:23,466 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:23,466 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., storeName=ae951cc3e18af3209220d547b1b63947/C, priority=13, startTime=1732223663220; duration=0sec 2024-11-21T21:14:23,467 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:23,467 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae951cc3e18af3209220d547b1b63947:C 2024-11-21T21:14:23,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T21:14:23,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:23,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223723511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:23,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:23,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:23,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:23,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:23,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223723511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:23,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:23,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:23,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:23,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:23,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223723512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:23,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:23,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:23,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:23,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:23,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223723531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:23,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:23,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:23,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:23,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:23,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223723531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:23,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:23,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:23,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:23,539 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-21T21:14:23,540 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-21T21:14:23,542 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-21T21:14:23,542 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-21T21:14:23,542 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-21T21:14:23,542 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-21T21:14:23,544 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-21T21:14:23,544 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-21T21:14:23,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:23,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223723720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:23,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:23,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:23,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:23,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:23,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223723721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:23,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:23,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:23,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:23,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:23,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223723731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:23,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:23,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:23,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:23,742 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/653f2f5b1c094038a95f14af19f28bb7 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/653f2f5b1c094038a95f14af19f28bb7 2024-11-21T21:14:23,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:23,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223723740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:23,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:23,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223723741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:23,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:23,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:23,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:23,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:23,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:23,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:23,758 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae951cc3e18af3209220d547b1b63947/A of ae951cc3e18af3209220d547b1b63947 into 653f2f5b1c094038a95f14af19f28bb7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:14:23,758 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:23,758 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., storeName=ae951cc3e18af3209220d547b1b63947/A, priority=13, startTime=1732223663219; duration=0sec 2024-11-21T21:14:23,759 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:23,759 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae951cc3e18af3209220d547b1b63947:A 2024-11-21T21:14:23,834 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/17f81127c74048aba81497f87bd5e668 2024-11-21T21:14:23,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/f1051ed0626649619563f573be753400 is 50, key is test_row_0/C:col10/1732223662977/Put/seqid=0 2024-11-21T21:14:23,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741860_1036 (size=12001) 2024-11-21T21:14:23,907 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/f1051ed0626649619563f573be753400 2024-11-21T21:14:23,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/f0e1c06ad55b40a9860bb148af5a70d0 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/f0e1c06ad55b40a9860bb148af5a70d0 2024-11-21T21:14:23,942 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/f0e1c06ad55b40a9860bb148af5a70d0, entries=150, sequenceid=103, filesize=11.7 K 2024-11-21T21:14:23,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/17f81127c74048aba81497f87bd5e668 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/17f81127c74048aba81497f87bd5e668 2024-11-21T21:14:23,963 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/17f81127c74048aba81497f87bd5e668, entries=150, sequenceid=103, filesize=11.7 K 2024-11-21T21:14:23,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/f1051ed0626649619563f573be753400 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/f1051ed0626649619563f573be753400 2024-11-21T21:14:23,979 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/f1051ed0626649619563f573be753400, entries=150, sequenceid=103, filesize=11.7 K 2024-11-21T21:14:23,981 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~64.85 KB/66410, heapSize ~184.05 KB/188464, currentSize=114.05 KB/116790 for ae951cc3e18af3209220d547b1b63947 in 663ms, sequenceid=103, compaction requested=false 2024-11-21T21:14:23,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:23,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:23,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-21T21:14:23,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-21T21:14:23,989 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-21T21:14:23,989 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1330 sec 2024-11-21T21:14:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T21:14:23,996 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.1490 sec 2024-11-21T21:14:24,046 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732223663378, max timestamp=1732223664035Num uniques 859; 2024-11-21T21:14:24,046 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732223663378, max timestamp=1732223664032Num uniques 859; 2024-11-21T21:14:24,046 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T21:14:24,046 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T21:14:24,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on ae951cc3e18af3209220d547b1b63947 2024-11-21T21:14:24,049 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae951cc3e18af3209220d547b1b63947 3/3 column families, dataSize=64.85 KB heapSize=184.09 KB 2024-11-21T21:14:24,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=A 2024-11-21T21:14:24,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:14:24,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=B 2024-11-21T21:14:24,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:14:24,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=C 2024-11-21T21:14:24,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:14:24,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/aa6e505ecf4c4193827417ef67a2ec19 is 50, key is test_row_0/A:col10/1732223664035/Put/seqid=0 2024-11-21T21:14:24,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741861_1037 (size=12001) 2024-11-21T21:14:24,094 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/aa6e505ecf4c4193827417ef67a2ec19 2024-11-21T21:14:24,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:24,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223724093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:24,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:24,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:24,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:24,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:24,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223724093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:24,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:24,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:24,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:24,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:24,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223724096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:24,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:24,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:24,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:24,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:24,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223724098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:24,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/e8fea30b633a4cd18f1b3511c309116f is 50, key is test_row_0/B:col10/1732223664035/Put/seqid=0 2024-11-21T21:14:24,111 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:24,112 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:24,112 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:24,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:24,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223724099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:24,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:24,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:24,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:24,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741862_1038 (size=12001) 2024-11-21T21:14:24,125 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/e8fea30b633a4cd18f1b3511c309116f 2024-11-21T21:14:24,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/ae136ccd0bde44c49c7ccd1c44e1496b is 50, key is test_row_0/C:col10/1732223664035/Put/seqid=0 2024-11-21T21:14:24,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741863_1039 (size=12001) 2024-11-21T21:14:24,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/ae136ccd0bde44c49c7ccd1c44e1496b 2024-11-21T21:14:24,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/aa6e505ecf4c4193827417ef67a2ec19 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/aa6e505ecf4c4193827417ef67a2ec19 2024-11-21T21:14:24,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/aa6e505ecf4c4193827417ef67a2ec19, entries=150, sequenceid=128, filesize=11.7 K 2024-11-21T21:14:24,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/e8fea30b633a4cd18f1b3511c309116f as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/e8fea30b633a4cd18f1b3511c309116f 2024-11-21T21:14:24,212 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/e8fea30b633a4cd18f1b3511c309116f, entries=150, sequenceid=128, filesize=11.7 K 2024-11-21T21:14:24,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/ae136ccd0bde44c49c7ccd1c44e1496b as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/ae136ccd0bde44c49c7ccd1c44e1496b 2024-11-21T21:14:24,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:24,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:24,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223724210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:24,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:24,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223724221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:24,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:24,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:24,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:24,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:24,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:24,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:24,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:24,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223724222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:24,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:24,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:24,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:24,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:24,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223724222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:24,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:24,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:24,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:24,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223724210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:24,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:24,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:24,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:24,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/ae136ccd0bde44c49c7ccd1c44e1496b, entries=150, sequenceid=128, filesize=11.7 K 2024-11-21T21:14:24,237 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~64.85 KB/66410, heapSize ~184.05 KB/188464, currentSize=114.05 KB/116790 for ae951cc3e18af3209220d547b1b63947 in 188ms, sequenceid=128, compaction requested=true 2024-11-21T21:14:24,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:24,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae951cc3e18af3209220d547b1b63947:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:14:24,238 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:14:24,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:24,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae951cc3e18af3209220d547b1b63947:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:14:24,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:24,238 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:14:24,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae951cc3e18af3209220d547b1b63947:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:14:24,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:14:24,241 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:14:24,241 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): ae951cc3e18af3209220d547b1b63947/B is initiating minor compaction (all files) 2024-11-21T21:14:24,242 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae951cc3e18af3209220d547b1b63947/B in TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:24,242 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/3cedb04748894fd1bd65a4f4db283444, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/17f81127c74048aba81497f87bd5e668, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/e8fea30b633a4cd18f1b3511c309116f] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp, totalSize=35.4 K 2024-11-21T21:14:24,242 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:14:24,242 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): ae951cc3e18af3209220d547b1b63947/A is initiating minor compaction (all files) 2024-11-21T21:14:24,242 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae951cc3e18af3209220d547b1b63947/A in TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:24,243 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/653f2f5b1c094038a95f14af19f28bb7, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/f0e1c06ad55b40a9860bb148af5a70d0, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/aa6e505ecf4c4193827417ef67a2ec19] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp, totalSize=35.4 K 2024-11-21T21:14:24,244 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 653f2f5b1c094038a95f14af19f28bb7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732223662582 2024-11-21T21:14:24,244 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 3cedb04748894fd1bd65a4f4db283444, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732223662582 2024-11-21T21:14:24,245 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0e1c06ad55b40a9860bb148af5a70d0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732223662977 2024-11-21T21:14:24,245 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 17f81127c74048aba81497f87bd5e668, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732223662977 2024-11-21T21:14:24,246 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa6e505ecf4c4193827417ef67a2ec19, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732223663394 2024-11-21T21:14:24,246 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting e8fea30b633a4cd18f1b3511c309116f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732223663394 2024-11-21T21:14:24,271 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae951cc3e18af3209220d547b1b63947#A#compaction#27 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:14:24,272 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/1996f79146a44a26aa1f4e8b0a96b5f7 is 50, key is test_row_0/A:col10/1732223664035/Put/seqid=0 2024-11-21T21:14:24,279 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae951cc3e18af3209220d547b1b63947#B#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:14:24,280 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/8808ec075f354e62895ce342b275cf0f is 50, key is test_row_0/B:col10/1732223664035/Put/seqid=0 2024-11-21T21:14:24,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741864_1040 (size=12309) 2024-11-21T21:14:24,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741865_1041 (size=12309) 2024-11-21T21:14:24,439 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae951cc3e18af3209220d547b1b63947 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-21T21:14:24,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on ae951cc3e18af3209220d547b1b63947 2024-11-21T21:14:24,440 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732223664090, max timestamp=1732223664434Num uniques 859; 2024-11-21T21:14:24,440 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T21:14:24,440 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732223664090, max timestamp=1732223664434Num uniques 859; 2024-11-21T21:14:24,440 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T21:14:24,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=A 2024-11-21T21:14:24,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:14:24,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=B 2024-11-21T21:14:24,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:14:24,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=C 2024-11-21T21:14:24,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:14:24,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/d48b357108a34a5083b680e6f9130b86 is 50, key is test_row_0/A:col10/1732223664438/Put/seqid=0 2024-11-21T21:14:24,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741866_1042 (size=14541) 2024-11-21T21:14:24,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:24,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223724476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:24,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:24,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223724490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:24,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:24,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:24,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223724493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:24,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:24,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:24,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:24,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:24,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223724493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:24,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:24,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:24,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:24,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223724494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:24,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:24,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:24,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:24,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:24,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:24,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223724600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:24,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:24,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223724601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:24,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:24,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:24,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:24,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:24,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:24,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:24,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:24,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223724611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:24,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:24,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:24,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:24,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:24,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223724611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:24,620 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:24,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:24,620 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:24,620 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:24,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223724611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:24,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:24,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:24,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:24,715 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/1996f79146a44a26aa1f4e8b0a96b5f7 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/1996f79146a44a26aa1f4e8b0a96b5f7 2024-11-21T21:14:24,719 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/8808ec075f354e62895ce342b275cf0f as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/8808ec075f354e62895ce342b275cf0f 2024-11-21T21:14:24,729 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae951cc3e18af3209220d547b1b63947/A of ae951cc3e18af3209220d547b1b63947 into 1996f79146a44a26aa1f4e8b0a96b5f7(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:14:24,729 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:24,729 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., storeName=ae951cc3e18af3209220d547b1b63947/A, priority=13, startTime=1732223664237; duration=0sec 2024-11-21T21:14:24,729 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:14:24,729 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae951cc3e18af3209220d547b1b63947:A 2024-11-21T21:14:24,729 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:14:24,731 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:14:24,731 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): ae951cc3e18af3209220d547b1b63947/C is initiating minor compaction (all files) 2024-11-21T21:14:24,732 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae951cc3e18af3209220d547b1b63947/C in TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:24,732 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/f9090e80448d436ba2c6fd7b5683ab0e, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/f1051ed0626649619563f573be753400, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/ae136ccd0bde44c49c7ccd1c44e1496b] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp, totalSize=35.4 K 2024-11-21T21:14:24,733 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9090e80448d436ba2c6fd7b5683ab0e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732223662582 2024-11-21T21:14:24,733 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae951cc3e18af3209220d547b1b63947/B of ae951cc3e18af3209220d547b1b63947 into 8808ec075f354e62895ce342b275cf0f(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:14:24,733 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:24,734 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., storeName=ae951cc3e18af3209220d547b1b63947/B, priority=13, startTime=1732223664238; duration=0sec 2024-11-21T21:14:24,734 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1051ed0626649619563f573be753400, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732223662977 2024-11-21T21:14:24,734 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:24,734 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae951cc3e18af3209220d547b1b63947:B 2024-11-21T21:14:24,736 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae136ccd0bde44c49c7ccd1c44e1496b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732223663394 2024-11-21T21:14:24,750 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae951cc3e18af3209220d547b1b63947#C#compaction#30 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:14:24,751 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/9b9b3e7b67f648a0b2ec8f3cd8d0b836 is 50, key is test_row_0/C:col10/1732223664035/Put/seqid=0 2024-11-21T21:14:24,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741867_1043 (size=12309) 2024-11-21T21:14:24,794 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/9b9b3e7b67f648a0b2ec8f3cd8d0b836 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/9b9b3e7b67f648a0b2ec8f3cd8d0b836 2024-11-21T21:14:24,806 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae951cc3e18af3209220d547b1b63947/C of ae951cc3e18af3209220d547b1b63947 into 9b9b3e7b67f648a0b2ec8f3cd8d0b836(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:14:24,807 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:24,807 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., storeName=ae951cc3e18af3209220d547b1b63947/C, priority=13, startTime=1732223664238; duration=0sec 2024-11-21T21:14:24,807 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:24,807 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae951cc3e18af3209220d547b1b63947:C 2024-11-21T21:14:24,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:24,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:24,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223724811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:24,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223724810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:24,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:24,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:24,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:24,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:24,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:24,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:24,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:24,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223724830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:24,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:24,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:24,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:24,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:24,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223724830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:24,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:24,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:24,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:24,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:24,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223724830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:24,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:24,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:24,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:24,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/d48b357108a34a5083b680e6f9130b86 2024-11-21T21:14:24,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/5f7dd2823e6c429da117b3fbb9a9535e is 50, key is test_row_0/B:col10/1732223664438/Put/seqid=0 2024-11-21T21:14:24,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741868_1044 (size=12151) 2024-11-21T21:14:24,912 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/5f7dd2823e6c429da117b3fbb9a9535e 2024-11-21T21:14:24,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/e3725e3657ca4a8a94ac66dcf82d3cd0 is 50, key is test_row_0/C:col10/1732223664438/Put/seqid=0 2024-11-21T21:14:24,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741869_1045 (size=12151) 2024-11-21T21:14:25,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T21:14:25,000 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:14:25,001 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:14:25,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-21T21:14:25,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T21:14:25,006 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:14:25,008 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:14:25,012 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:14:25,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T21:14:25,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:25,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223725130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:25,132 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:25,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:25,132 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:25,132 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:25,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223725131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:25,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:25,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:25,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:25,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:25,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223725140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:25,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:25,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:25,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:25,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:25,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223725141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:25,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:25,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:25,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:25,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223725143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:25,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:25,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:25,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:25,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:25,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T21:14:25,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:25,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:25,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:25,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:25,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:25,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:25,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T21:14:25,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T21:14:25,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:25,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:25,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:25,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:25,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:25,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:25,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/e3725e3657ca4a8a94ac66dcf82d3cd0 2024-11-21T21:14:25,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/d48b357108a34a5083b680e6f9130b86 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/d48b357108a34a5083b680e6f9130b86 2024-11-21T21:14:25,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/d48b357108a34a5083b680e6f9130b86, entries=200, sequenceid=151, filesize=14.2 K 2024-11-21T21:14:25,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/5f7dd2823e6c429da117b3fbb9a9535e as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/5f7dd2823e6c429da117b3fbb9a9535e 2024-11-21T21:14:25,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/5f7dd2823e6c429da117b3fbb9a9535e, entries=150, sequenceid=151, filesize=11.9 K 2024-11-21T21:14:25,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/e3725e3657ca4a8a94ac66dcf82d3cd0 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/e3725e3657ca4a8a94ac66dcf82d3cd0 2024-11-21T21:14:25,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/e3725e3657ca4a8a94ac66dcf82d3cd0, entries=150, sequenceid=151, filesize=11.9 K 2024-11-21T21:14:25,388 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~71.56 KB/73280, heapSize ~203.97 KB/208864, currentSize=107.34 KB/109920 for ae951cc3e18af3209220d547b1b63947 in 948ms, sequenceid=151, compaction requested=false 2024-11-21T21:14:25,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:25,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T21:14:25,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:25,479 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing ae951cc3e18af3209220d547b1b63947 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-21T21:14:25,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=A 2024-11-21T21:14:25,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:25,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=B 2024-11-21T21:14:25,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:25,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=C 2024-11-21T21:14:25,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:25,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/d296505f965041929408b93f3b2415a4 is 50, key is test_row_0/A:col10/1732223664490/Put/seqid=0 2024-11-21T21:14:25,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741870_1046 (size=12151) 2024-11-21T21:14:25,511 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/d296505f965041929408b93f3b2415a4 2024-11-21T21:14:25,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/f854f6c9ae7349358543755116602d4a is 50, key is test_row_0/B:col10/1732223664490/Put/seqid=0 2024-11-21T21:14:25,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741871_1047 (size=12151) 2024-11-21T21:14:25,548 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/f854f6c9ae7349358543755116602d4a 2024-11-21T21:14:25,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/32d12867be1346d1abea1c360b772414 is 50, key is test_row_0/C:col10/1732223664490/Put/seqid=0 2024-11-21T21:14:25,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741872_1048 (size=12151) 2024-11-21T21:14:25,573 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/32d12867be1346d1abea1c360b772414 2024-11-21T21:14:25,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/d296505f965041929408b93f3b2415a4 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/d296505f965041929408b93f3b2415a4 2024-11-21T21:14:25,610 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/d296505f965041929408b93f3b2415a4, entries=150, sequenceid=173, filesize=11.9 K 2024-11-21T21:14:25,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/f854f6c9ae7349358543755116602d4a as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/f854f6c9ae7349358543755116602d4a 2024-11-21T21:14:25,624 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/f854f6c9ae7349358543755116602d4a, entries=150, sequenceid=173, filesize=11.9 K 2024-11-21T21:14:25,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/32d12867be1346d1abea1c360b772414 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/32d12867be1346d1abea1c360b772414 2024-11-21T21:14:25,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T21:14:25,641 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/32d12867be1346d1abea1c360b772414, entries=150, sequenceid=173, filesize=11.9 K 2024-11-21T21:14:25,643 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=6.71 KB/6870 for ae951cc3e18af3209220d547b1b63947 in 164ms, sequenceid=173, compaction requested=true 2024-11-21T21:14:25,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:25,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:25,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-21T21:14:25,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-21T21:14:25,655 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-21T21:14:25,655 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 641 msec 2024-11-21T21:14:25,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on ae951cc3e18af3209220d547b1b63947 2024-11-21T21:14:25,656 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae951cc3e18af3209220d547b1b63947 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T21:14:25,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=A 2024-11-21T21:14:25,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:25,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=B 2024-11-21T21:14:25,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:25,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=C 2024-11-21T21:14:25,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:25,661 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 657 msec 2024-11-21T21:14:25,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/240e1bcd45df4ae1a0e29f091ab8813a is 50, key is test_row_0/A:col10/1732223665646/Put/seqid=0 2024-11-21T21:14:25,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741873_1049 (size=14541) 2024-11-21T21:14:25,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732223665707, max timestamp=1732223665724Num uniques 859; 2024-11-21T21:14:25,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732223665707, max timestamp=1732223665724Num uniques 859; 2024-11-21T21:14:25,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T21:14:25,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T21:14:25,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732223665707, max timestamp=1732223665724Num uniques 859; 2024-11-21T21:14:25,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T21:14:25,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:25,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223725753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:25,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:25,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223725756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:25,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:25,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223725758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:25,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:25,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:25,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:25,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:25,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:25,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:25,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223725761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:25,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:25,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:25,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:25,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:25,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:25,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:25,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:25,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:25,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223725770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:25,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:25,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:25,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:25,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:25,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223725871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:25,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:25,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:25,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:25,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:25,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223725880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:25,884 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:25,884 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:25,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:25,884 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:25,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223725880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:25,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:25,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:25,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:25,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:25,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223725880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:25,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:25,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223725880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:25,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:25,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:25,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:25,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:25,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:25,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:26,079 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/240e1bcd45df4ae1a0e29f091ab8813a 2024-11-21T21:14:26,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:26,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223726080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:26,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:26,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:26,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:26,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:26,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:26,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223726091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:26,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:26,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:26,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:26,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:26,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223726091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:26,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:26,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:26,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223726091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:26,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:26,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:26,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:26,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:26,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/e7ad00743dbe44bea1e0b7cfe4aeea92 is 50, key is test_row_0/B:col10/1732223665646/Put/seqid=0 2024-11-21T21:14:26,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:26,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223726091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:26,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:26,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:26,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:26,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741874_1050 (size=12151) 2024-11-21T21:14:26,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T21:14:26,141 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:14:26,142 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:14:26,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-21T21:14:26,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T21:14:26,144 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:14:26,145 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:14:26,145 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:14:26,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T21:14:26,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-21T21:14:26,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:26,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:26,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:26,299 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:26,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:26,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:26,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:26,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223726390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:26,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:26,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:26,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:26,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:26,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223726400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:26,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:26,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:26,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:26,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:26,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223726400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:26,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:26,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:26,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:26,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:26,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223726400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:26,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:26,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:26,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:26,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223726411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:26,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:26,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:26,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:26,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-21T21:14:26,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:26,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:26,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:26,452 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:26,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:26,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T21:14:26,512 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/e7ad00743dbe44bea1e0b7cfe4aeea92 2024-11-21T21:14:26,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/63196fb0b0ea475ba60295f9447ebb22 is 50, key is test_row_0/C:col10/1732223665646/Put/seqid=0 2024-11-21T21:14:26,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741875_1051 (size=12151) 2024-11-21T21:14:26,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-21T21:14:26,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:26,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:26,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:26,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:26,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:26,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:26,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-21T21:14:26,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:26,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:26,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:26,761 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:26,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:26,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:26,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T21:14:26,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:26,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223726900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:26,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:26,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:26,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:26,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-21T21:14:26,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:26,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:26,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:26,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:26,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:26,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:26,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:26,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223726911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:26,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:26,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:26,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:26,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:26,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223726911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:26,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:26,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:26,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:26,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:26,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223726920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:26,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:26,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:26,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:26,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:26,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223726920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:26,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:26,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:26,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:26,931 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/63196fb0b0ea475ba60295f9447ebb22 2024-11-21T21:14:26,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/240e1bcd45df4ae1a0e29f091ab8813a as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/240e1bcd45df4ae1a0e29f091ab8813a 2024-11-21T21:14:26,955 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/240e1bcd45df4ae1a0e29f091ab8813a, entries=200, sequenceid=183, filesize=14.2 K 2024-11-21T21:14:26,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/e7ad00743dbe44bea1e0b7cfe4aeea92 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/e7ad00743dbe44bea1e0b7cfe4aeea92 2024-11-21T21:14:26,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/e7ad00743dbe44bea1e0b7cfe4aeea92, entries=150, sequenceid=183, filesize=11.9 K 2024-11-21T21:14:26,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/63196fb0b0ea475ba60295f9447ebb22 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/63196fb0b0ea475ba60295f9447ebb22 2024-11-21T21:14:26,989 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/63196fb0b0ea475ba60295f9447ebb22, entries=150, sequenceid=183, filesize=11.9 K 2024-11-21T21:14:26,990 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for ae951cc3e18af3209220d547b1b63947 in 1334ms, sequenceid=183, compaction requested=true 2024-11-21T21:14:26,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:26,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae951cc3e18af3209220d547b1b63947:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:14:26,991 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:14:26,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:26,991 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:14:26,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae951cc3e18af3209220d547b1b63947:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:14:26,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:26,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae951cc3e18af3209220d547b1b63947:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:14:26,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:14:26,993 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48762 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:14:26,993 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): ae951cc3e18af3209220d547b1b63947/B is initiating minor compaction (all files) 2024-11-21T21:14:26,993 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae951cc3e18af3209220d547b1b63947/B in TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:26,994 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/8808ec075f354e62895ce342b275cf0f, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/5f7dd2823e6c429da117b3fbb9a9535e, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/f854f6c9ae7349358543755116602d4a, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/e7ad00743dbe44bea1e0b7cfe4aeea92] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp, totalSize=47.6 K 2024-11-21T21:14:26,995 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53542 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:14:26,995 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): ae951cc3e18af3209220d547b1b63947/A is initiating minor compaction (all files) 2024-11-21T21:14:26,995 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae951cc3e18af3209220d547b1b63947/A in TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:26,996 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/1996f79146a44a26aa1f4e8b0a96b5f7, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/d48b357108a34a5083b680e6f9130b86, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/d296505f965041929408b93f3b2415a4, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/240e1bcd45df4ae1a0e29f091ab8813a] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp, totalSize=52.3 K 2024-11-21T21:14:26,996 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 8808ec075f354e62895ce342b275cf0f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732223663394 2024-11-21T21:14:26,996 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f7dd2823e6c429da117b3fbb9a9535e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732223664090 2024-11-21T21:14:26,996 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1996f79146a44a26aa1f4e8b0a96b5f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732223663394 2024-11-21T21:14:26,997 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting f854f6c9ae7349358543755116602d4a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732223664485 2024-11-21T21:14:26,997 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting d48b357108a34a5083b680e6f9130b86, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732223664090 2024-11-21T21:14:26,997 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting e7ad00743dbe44bea1e0b7cfe4aeea92, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732223665646 2024-11-21T21:14:26,997 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting d296505f965041929408b93f3b2415a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732223664485 2024-11-21T21:14:26,998 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 240e1bcd45df4ae1a0e29f091ab8813a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732223665646 2024-11-21T21:14:27,028 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae951cc3e18af3209220d547b1b63947#A#compaction#39 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:14:27,029 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/62db0869e8a044738693d02913cbfa9a is 50, key is test_row_0/A:col10/1732223665646/Put/seqid=0 2024-11-21T21:14:27,034 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae951cc3e18af3209220d547b1b63947#B#compaction#40 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:14:27,035 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/7ed0b3b9fbb6466db0e2274a8f7b7db9 is 50, key is test_row_0/B:col10/1732223665646/Put/seqid=0 2024-11-21T21:14:27,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741876_1052 (size=12595) 2024-11-21T21:14:27,062 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/62db0869e8a044738693d02913cbfa9a as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/62db0869e8a044738693d02913cbfa9a 2024-11-21T21:14:27,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-21T21:14:27,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:27,071 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing ae951cc3e18af3209220d547b1b63947 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-21T21:14:27,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=A 2024-11-21T21:14:27,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:14:27,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=B 2024-11-21T21:14:27,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:14:27,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=C 2024-11-21T21:14:27,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:14:27,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741877_1053 (size=12595) 2024-11-21T21:14:27,081 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae951cc3e18af3209220d547b1b63947/A of ae951cc3e18af3209220d547b1b63947 into 62db0869e8a044738693d02913cbfa9a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:14:27,081 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:27,081 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., storeName=ae951cc3e18af3209220d547b1b63947/A, priority=12, startTime=1732223666990; duration=0sec 2024-11-21T21:14:27,081 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:14:27,081 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae951cc3e18af3209220d547b1b63947:A 2024-11-21T21:14:27,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/04f6461152e941ae8b8476242acd278b is 50, key is test_row_0/A:col10/1732223665749/Put/seqid=0 2024-11-21T21:14:27,081 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:14:27,085 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48762 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:14:27,085 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/7ed0b3b9fbb6466db0e2274a8f7b7db9 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/7ed0b3b9fbb6466db0e2274a8f7b7db9 2024-11-21T21:14:27,085 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): ae951cc3e18af3209220d547b1b63947/C is initiating minor compaction (all files) 2024-11-21T21:14:27,085 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae951cc3e18af3209220d547b1b63947/C in TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:27,086 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/9b9b3e7b67f648a0b2ec8f3cd8d0b836, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/e3725e3657ca4a8a94ac66dcf82d3cd0, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/32d12867be1346d1abea1c360b772414, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/63196fb0b0ea475ba60295f9447ebb22] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp, totalSize=47.6 K 2024-11-21T21:14:27,086 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b9b3e7b67f648a0b2ec8f3cd8d0b836, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732223663394 2024-11-21T21:14:27,087 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3725e3657ca4a8a94ac66dcf82d3cd0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732223664090 2024-11-21T21:14:27,087 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32d12867be1346d1abea1c360b772414, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732223664485 2024-11-21T21:14:27,088 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63196fb0b0ea475ba60295f9447ebb22, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732223665646 2024-11-21T21:14:27,097 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae951cc3e18af3209220d547b1b63947/B of ae951cc3e18af3209220d547b1b63947 into 7ed0b3b9fbb6466db0e2274a8f7b7db9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:14:27,097 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:27,097 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., storeName=ae951cc3e18af3209220d547b1b63947/B, priority=12, startTime=1732223666991; duration=0sec 2024-11-21T21:14:27,097 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:27,097 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae951cc3e18af3209220d547b1b63947:B 2024-11-21T21:14:27,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741878_1054 (size=12151) 2024-11-21T21:14:27,120 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/04f6461152e941ae8b8476242acd278b 2024-11-21T21:14:27,122 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae951cc3e18af3209220d547b1b63947#C#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:14:27,123 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/b8e77c80b7a34404b91c3a1e721aed08 is 50, key is test_row_0/C:col10/1732223665646/Put/seqid=0 2024-11-21T21:14:27,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/061b12adb0c74f1b802323cb6a85fc6d is 50, key is test_row_0/B:col10/1732223665749/Put/seqid=0 2024-11-21T21:14:27,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741879_1055 (size=12595) 2024-11-21T21:14:27,180 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/b8e77c80b7a34404b91c3a1e721aed08 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/b8e77c80b7a34404b91c3a1e721aed08 2024-11-21T21:14:27,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741880_1056 (size=12151) 2024-11-21T21:14:27,193 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/061b12adb0c74f1b802323cb6a85fc6d 2024-11-21T21:14:27,203 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae951cc3e18af3209220d547b1b63947/C of ae951cc3e18af3209220d547b1b63947 into b8e77c80b7a34404b91c3a1e721aed08(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:14:27,203 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:27,203 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., storeName=ae951cc3e18af3209220d547b1b63947/C, priority=12, startTime=1732223666991; duration=0sec 2024-11-21T21:14:27,203 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:27,203 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae951cc3e18af3209220d547b1b63947:C 2024-11-21T21:14:27,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/028d3d9dc29446a2a1e90b3a9f1426ca is 50, key is test_row_0/C:col10/1732223665749/Put/seqid=0 2024-11-21T21:14:27,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741881_1057 (size=12151) 2024-11-21T21:14:27,239 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/028d3d9dc29446a2a1e90b3a9f1426ca 2024-11-21T21:14:27,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/04f6461152e941ae8b8476242acd278b as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/04f6461152e941ae8b8476242acd278b 2024-11-21T21:14:27,262 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/04f6461152e941ae8b8476242acd278b, entries=150, sequenceid=219, filesize=11.9 K 2024-11-21T21:14:27,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/061b12adb0c74f1b802323cb6a85fc6d as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/061b12adb0c74f1b802323cb6a85fc6d 2024-11-21T21:14:27,270 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/061b12adb0c74f1b802323cb6a85fc6d, entries=150, sequenceid=219, filesize=11.9 K 2024-11-21T21:14:27,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/028d3d9dc29446a2a1e90b3a9f1426ca as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/028d3d9dc29446a2a1e90b3a9f1426ca 2024-11-21T21:14:27,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T21:14:27,281 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/028d3d9dc29446a2a1e90b3a9f1426ca, entries=150, sequenceid=219, filesize=11.9 K 2024-11-21T21:14:27,283 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=0 B/0 for ae951cc3e18af3209220d547b1b63947 in 212ms, sequenceid=219, compaction requested=false 2024-11-21T21:14:27,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:27,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:27,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-21T21:14:27,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-21T21:14:27,289 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-21T21:14:27,289 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1400 sec 2024-11-21T21:14:27,296 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.1510 sec 2024-11-21T21:14:27,936 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae951cc3e18af3209220d547b1b63947 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T21:14:27,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=A 2024-11-21T21:14:27,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:27,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=B 2024-11-21T21:14:27,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:27,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=C 2024-11-21T21:14:27,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:27,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on ae951cc3e18af3209220d547b1b63947 2024-11-21T21:14:27,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/c7d27b799d784851997079ab12b90a82 is 50, key is test_row_0/A:col10/1732223667933/Put/seqid=0 2024-11-21T21:14:27,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741882_1058 (size=9757) 2024-11-21T21:14:27,958 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/c7d27b799d784851997079ab12b90a82 2024-11-21T21:14:27,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/235fcd71c5cc4b299ab2c4883baf5ece is 50, key is test_row_0/B:col10/1732223667933/Put/seqid=0 2024-11-21T21:14:28,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732223667948, max timestamp=1732223667997Num uniques 859; 2024-11-21T21:14:28,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-21T21:14:28,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732223667948, max timestamp=1732223667997Num uniques 859; 2024-11-21T21:14:28,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-21T21:14:28,007 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732223667989, max timestamp=1732223667997Num uniques 859; 2024-11-21T21:14:28,007 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-21T21:14:28,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741883_1059 (size=9757) 2024-11-21T21:14:28,016 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/235fcd71c5cc4b299ab2c4883baf5ece 2024-11-21T21:14:28,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/a910630e4c784ce9be456a46afa0c443 is 50, key is test_row_0/C:col10/1732223667933/Put/seqid=0 2024-11-21T21:14:28,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741884_1060 (size=9757) 2024-11-21T21:14:28,053 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/a910630e4c784ce9be456a46afa0c443 2024-11-21T21:14:28,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:28,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:28,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223728047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:28,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223728047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:28,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:28,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:28,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:28,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223728050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:28,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:28,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223728053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:28,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:28,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:28,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:28,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223728054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:28,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/c7d27b799d784851997079ab12b90a82 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/c7d27b799d784851997079ab12b90a82 2024-11-21T21:14:28,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:28,078 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/c7d27b799d784851997079ab12b90a82, entries=100, sequenceid=231, filesize=9.5 K 2024-11-21T21:14:28,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/235fcd71c5cc4b299ab2c4883baf5ece as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/235fcd71c5cc4b299ab2c4883baf5ece 2024-11-21T21:14:28,095 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/235fcd71c5cc4b299ab2c4883baf5ece, entries=100, sequenceid=231, filesize=9.5 K 2024-11-21T21:14:28,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/a910630e4c784ce9be456a46afa0c443 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/a910630e4c784ce9be456a46afa0c443 2024-11-21T21:14:28,113 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/a910630e4c784ce9be456a46afa0c443, entries=100, sequenceid=231, filesize=9.5 K 2024-11-21T21:14:28,116 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=133.78 KB/136995 for ae951cc3e18af3209220d547b1b63947 in 180ms, sequenceid=231, compaction requested=true 2024-11-21T21:14:28,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:28,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae951cc3e18af3209220d547b1b63947:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:14:28,117 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:14:28,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:28,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae951cc3e18af3209220d547b1b63947:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:14:28,117 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:14:28,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:28,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae951cc3e18af3209220d547b1b63947:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:14:28,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:14:28,119 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:14:28,119 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): ae951cc3e18af3209220d547b1b63947/B is initiating minor compaction (all files) 2024-11-21T21:14:28,119 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae951cc3e18af3209220d547b1b63947/B in TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:28,120 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/7ed0b3b9fbb6466db0e2274a8f7b7db9, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/061b12adb0c74f1b802323cb6a85fc6d, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/235fcd71c5cc4b299ab2c4883baf5ece] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp, totalSize=33.7 K 2024-11-21T21:14:28,120 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:14:28,120 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): ae951cc3e18af3209220d547b1b63947/A is initiating minor compaction (all files) 2024-11-21T21:14:28,120 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae951cc3e18af3209220d547b1b63947/A in TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:28,120 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/62db0869e8a044738693d02913cbfa9a, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/04f6461152e941ae8b8476242acd278b, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/c7d27b799d784851997079ab12b90a82] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp, totalSize=33.7 K 2024-11-21T21:14:28,121 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ed0b3b9fbb6466db0e2274a8f7b7db9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732223665646 2024-11-21T21:14:28,122 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62db0869e8a044738693d02913cbfa9a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732223665646 2024-11-21T21:14:28,122 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 061b12adb0c74f1b802323cb6a85fc6d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732223665744 2024-11-21T21:14:28,122 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04f6461152e941ae8b8476242acd278b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732223665744 2024-11-21T21:14:28,123 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 235fcd71c5cc4b299ab2c4883baf5ece, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732223667933 2024-11-21T21:14:28,123 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7d27b799d784851997079ab12b90a82, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732223667933 2024-11-21T21:14:28,151 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae951cc3e18af3209220d547b1b63947#B#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:14:28,152 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/40724a1d8d9d4bcb9c4e8fb6e8e6be4e is 50, key is test_row_0/B:col10/1732223667933/Put/seqid=0 2024-11-21T21:14:28,160 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae951cc3e18af3209220d547b1b63947#A#compaction#49 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:14:28,160 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/221e4956b19343c78dcd8b154be99f00 is 50, key is test_row_0/A:col10/1732223667933/Put/seqid=0 2024-11-21T21:14:28,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on ae951cc3e18af3209220d547b1b63947 2024-11-21T21:14:28,170 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae951cc3e18af3209220d547b1b63947 3/3 column families, dataSize=140.49 KB heapSize=394.62 KB 2024-11-21T21:14:28,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=A 2024-11-21T21:14:28,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:14:28,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=B 2024-11-21T21:14:28,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:14:28,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=C 2024-11-21T21:14:28,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:14:28,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:28,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223728178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:28,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:28,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:28,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223728181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:28,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:28,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:28,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223728182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:28,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/0b18fb4120ae4f429df4263e21e98105 is 50, key is test_row_0/A:col10/1732223667997/Put/seqid=0 2024-11-21T21:14:28,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:28,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223728185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:28,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:28,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:28,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:28,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223728192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:28,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:28,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741886_1062 (size=12697) 2024-11-21T21:14:28,230 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/221e4956b19343c78dcd8b154be99f00 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/221e4956b19343c78dcd8b154be99f00 2024-11-21T21:14:28,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741885_1061 (size=12697) 2024-11-21T21:14:28,258 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae951cc3e18af3209220d547b1b63947/A of ae951cc3e18af3209220d547b1b63947 into 221e4956b19343c78dcd8b154be99f00(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:14:28,258 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:28,259 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., storeName=ae951cc3e18af3209220d547b1b63947/A, priority=13, startTime=1732223668117; duration=0sec 2024-11-21T21:14:28,259 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:14:28,259 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae951cc3e18af3209220d547b1b63947:A 2024-11-21T21:14:28,259 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:14:28,261 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:14:28,261 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): ae951cc3e18af3209220d547b1b63947/C is initiating minor compaction (all files) 2024-11-21T21:14:28,261 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae951cc3e18af3209220d547b1b63947/C in TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:28,261 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/b8e77c80b7a34404b91c3a1e721aed08, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/028d3d9dc29446a2a1e90b3a9f1426ca, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/a910630e4c784ce9be456a46afa0c443] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp, totalSize=33.7 K 2024-11-21T21:14:28,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741887_1063 (size=12251) 2024-11-21T21:14:28,265 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/40724a1d8d9d4bcb9c4e8fb6e8e6be4e as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/40724a1d8d9d4bcb9c4e8fb6e8e6be4e 2024-11-21T21:14:28,265 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/0b18fb4120ae4f429df4263e21e98105 2024-11-21T21:14:28,266 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8e77c80b7a34404b91c3a1e721aed08, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732223665646 2024-11-21T21:14:28,267 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 028d3d9dc29446a2a1e90b3a9f1426ca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732223665744 2024-11-21T21:14:28,270 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting a910630e4c784ce9be456a46afa0c443, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732223667933 2024-11-21T21:14:28,274 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae951cc3e18af3209220d547b1b63947/B of ae951cc3e18af3209220d547b1b63947 into 40724a1d8d9d4bcb9c4e8fb6e8e6be4e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:14:28,275 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:28,275 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., storeName=ae951cc3e18af3209220d547b1b63947/B, priority=13, startTime=1732223668117; duration=0sec 2024-11-21T21:14:28,275 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:28,275 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae951cc3e18af3209220d547b1b63947:B 2024-11-21T21:14:28,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/fddac71f19a4480f86558a1d27e46f84 is 50, key is test_row_0/B:col10/1732223667997/Put/seqid=0 2024-11-21T21:14:28,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T21:14:28,292 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:14:28,293 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:14:28,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-21T21:14:28,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T21:14:28,296 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:14:28,297 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:14:28,298 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:14:28,299 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae951cc3e18af3209220d547b1b63947#C#compaction#52 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:14:28,300 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/593633d8540048baa886a565bdd368ec is 50, key is test_row_0/C:col10/1732223667933/Put/seqid=0 2024-11-21T21:14:28,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:28,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223728300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:28,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:28,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:28,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:28,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223728311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:28,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223728311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:28,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:28,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223728311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:28,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:28,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:28,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223728311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:28,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:28,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:28,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:28,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741888_1064 (size=12251) 2024-11-21T21:14:28,340 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/fddac71f19a4480f86558a1d27e46f84 2024-11-21T21:14:28,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741889_1065 (size=12697) 2024-11-21T21:14:28,377 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/eae3b0e9ff0046e79dcdd787b380850b is 50, key is test_row_0/C:col10/1732223667997/Put/seqid=0 2024-11-21T21:14:28,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T21:14:28,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741890_1066 (size=12251) 2024-11-21T21:14:28,417 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/eae3b0e9ff0046e79dcdd787b380850b 2024-11-21T21:14:28,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/0b18fb4120ae4f429df4263e21e98105 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/0b18fb4120ae4f429df4263e21e98105 2024-11-21T21:14:28,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/0b18fb4120ae4f429df4263e21e98105, entries=150, sequenceid=267, filesize=12.0 K 2024-11-21T21:14:28,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/fddac71f19a4480f86558a1d27e46f84 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/fddac71f19a4480f86558a1d27e46f84 2024-11-21T21:14:28,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T21:14:28,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:28,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:28,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:28,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:28,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:28,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:28,463 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/fddac71f19a4480f86558a1d27e46f84, entries=150, sequenceid=267, filesize=12.0 K 2024-11-21T21:14:28,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/eae3b0e9ff0046e79dcdd787b380850b as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/eae3b0e9ff0046e79dcdd787b380850b 2024-11-21T21:14:28,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/eae3b0e9ff0046e79dcdd787b380850b, entries=150, sequenceid=267, filesize=12.0 K 2024-11-21T21:14:28,490 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.49 KB/143865, heapSize ~394.57 KB/404040, currentSize=40.25 KB/41220 for ae951cc3e18af3209220d547b1b63947 in 320ms, sequenceid=267, compaction requested=false 2024-11-21T21:14:28,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:28,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on ae951cc3e18af3209220d547b1b63947 2024-11-21T21:14:28,519 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae951cc3e18af3209220d547b1b63947 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T21:14:28,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=A 2024-11-21T21:14:28,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:28,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=B 2024-11-21T21:14:28,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:28,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=C 2024-11-21T21:14:28,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:28,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/e2b68f3ff39e4a08b964c5bf6f11ba42 is 50, key is test_row_0/A:col10/1732223668512/Put/seqid=0 2024-11-21T21:14:28,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741891_1067 (size=12301) 2024-11-21T21:14:28,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T21:14:28,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:28,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:28,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:28,609 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:28,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:28,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T21:14:28,612 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732223668582, max timestamp=1732223668604Num uniques 859; 2024-11-21T21:14:28,612 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732223668582, max timestamp=1732223668604Num uniques 859; 2024-11-21T21:14:28,612 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732223668582, max timestamp=1732223668604Num uniques 859; 2024-11-21T21:14:28,612 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T21:14:28,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T21:14:28,612 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T21:14:28,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:28,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:28,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223728649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:28,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:28,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:28,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223728651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:28,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:28,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:28,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223728653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:28,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:28,664 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223728657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:28,664 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,664 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:28,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:28,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:28,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223728663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:28,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:28,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T21:14:28,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:28,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:28,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:28,775 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:28,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:28,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:28,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:28,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223728771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:28,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:28,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223728771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:28,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:28,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:28,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223728771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:28,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:28,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223728771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:28,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:28,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:28,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:28,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:28,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223728773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:28,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:28,787 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/593633d8540048baa886a565bdd368ec as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/593633d8540048baa886a565bdd368ec 2024-11-21T21:14:28,798 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae951cc3e18af3209220d547b1b63947/C of ae951cc3e18af3209220d547b1b63947 into 593633d8540048baa886a565bdd368ec(size=12.4 K), total size for store is 24.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:14:28,798 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:28,798 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., storeName=ae951cc3e18af3209220d547b1b63947/C, priority=13, startTime=1732223668117; duration=0sec 2024-11-21T21:14:28,799 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:28,799 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae951cc3e18af3209220d547b1b63947:C 2024-11-21T21:14:28,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T21:14:28,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T21:14:28,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:28,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:28,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:28,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:28,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:28,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:28,949 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/e2b68f3ff39e4a08b964c5bf6f11ba42 2024-11-21T21:14:28,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/8d54d81eb3b84fa382954aaab774ed14 is 50, key is test_row_0/B:col10/1732223668512/Put/seqid=0 2024-11-21T21:14:28,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:28,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223728991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:28,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:28,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223728996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:28,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:28,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223728996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:28,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,999 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,999 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:28,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:28,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223728997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:28,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:28,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:28,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:28,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:29,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:29,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223729000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:29,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:29,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:29,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:29,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741892_1068 (size=12301) 2024-11-21T21:14:29,016 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/8d54d81eb3b84fa382954aaab774ed14 2024-11-21T21:14:29,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/6b65140ebaf2405ca0b387ff1340569b is 50, key is test_row_0/C:col10/1732223668512/Put/seqid=0 2024-11-21T21:14:29,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741893_1069 (size=12301) 2024-11-21T21:14:29,066 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/6b65140ebaf2405ca0b387ff1340569b 2024-11-21T21:14:29,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/e2b68f3ff39e4a08b964c5bf6f11ba42 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/e2b68f3ff39e4a08b964c5bf6f11ba42 2024-11-21T21:14:29,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T21:14:29,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:29,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:29,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:29,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:29,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:29,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:29,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/e2b68f3ff39e4a08b964c5bf6f11ba42, entries=150, sequenceid=279, filesize=12.0 K 2024-11-21T21:14:29,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/8d54d81eb3b84fa382954aaab774ed14 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/8d54d81eb3b84fa382954aaab774ed14 2024-11-21T21:14:29,099 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/8d54d81eb3b84fa382954aaab774ed14, entries=150, sequenceid=279, filesize=12.0 K 2024-11-21T21:14:29,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/6b65140ebaf2405ca0b387ff1340569b as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/6b65140ebaf2405ca0b387ff1340569b 2024-11-21T21:14:29,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/6b65140ebaf2405ca0b387ff1340569b, entries=150, sequenceid=279, filesize=12.0 K 2024-11-21T21:14:29,109 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for ae951cc3e18af3209220d547b1b63947 in 590ms, sequenceid=279, compaction requested=true 2024-11-21T21:14:29,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:29,110 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:14:29,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae951cc3e18af3209220d547b1b63947:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:14:29,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:29,110 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:14:29,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae951cc3e18af3209220d547b1b63947:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:14:29,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:29,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae951cc3e18af3209220d547b1b63947:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:14:29,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:14:29,111 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37249 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:14:29,111 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): ae951cc3e18af3209220d547b1b63947/B is initiating minor compaction (all files) 2024-11-21T21:14:29,111 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae951cc3e18af3209220d547b1b63947/B in TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:29,112 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/40724a1d8d9d4bcb9c4e8fb6e8e6be4e, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/fddac71f19a4480f86558a1d27e46f84, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/8d54d81eb3b84fa382954aaab774ed14] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp, totalSize=36.4 K 2024-11-21T21:14:29,112 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37249 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:14:29,112 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): ae951cc3e18af3209220d547b1b63947/A is initiating minor compaction (all files) 2024-11-21T21:14:29,112 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae951cc3e18af3209220d547b1b63947/A in TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:29,112 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/221e4956b19343c78dcd8b154be99f00, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/0b18fb4120ae4f429df4263e21e98105, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/e2b68f3ff39e4a08b964c5bf6f11ba42] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp, totalSize=36.4 K 2024-11-21T21:14:29,113 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 221e4956b19343c78dcd8b154be99f00, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732223665744 2024-11-21T21:14:29,113 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 40724a1d8d9d4bcb9c4e8fb6e8e6be4e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732223665744 2024-11-21T21:14:29,113 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting fddac71f19a4480f86558a1d27e46f84, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732223667997 2024-11-21T21:14:29,113 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b18fb4120ae4f429df4263e21e98105, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732223667997 2024-11-21T21:14:29,113 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d54d81eb3b84fa382954aaab774ed14, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732223668179 2024-11-21T21:14:29,114 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2b68f3ff39e4a08b964c5bf6f11ba42, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732223668179 2024-11-21T21:14:29,127 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae951cc3e18af3209220d547b1b63947#B#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:14:29,128 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/1089d7e37d2a4c3885839c69d7f760f6 is 50, key is test_row_0/B:col10/1732223668512/Put/seqid=0 2024-11-21T21:14:29,137 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae951cc3e18af3209220d547b1b63947#A#compaction#58 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:14:29,138 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/42032c7c555746a69a461e5e55f85f0a is 50, key is test_row_0/A:col10/1732223668512/Put/seqid=0 2024-11-21T21:14:29,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741894_1070 (size=12949) 2024-11-21T21:14:29,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741895_1071 (size=12949) 2024-11-21T21:14:29,171 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/42032c7c555746a69a461e5e55f85f0a as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/42032c7c555746a69a461e5e55f85f0a 2024-11-21T21:14:29,181 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae951cc3e18af3209220d547b1b63947/A of ae951cc3e18af3209220d547b1b63947 into 42032c7c555746a69a461e5e55f85f0a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:14:29,181 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:29,182 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., storeName=ae951cc3e18af3209220d547b1b63947/A, priority=13, startTime=1732223669110; duration=0sec 2024-11-21T21:14:29,182 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:14:29,182 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae951cc3e18af3209220d547b1b63947:A 2024-11-21T21:14:29,183 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:14:29,184 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37249 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:14:29,184 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): ae951cc3e18af3209220d547b1b63947/C is initiating minor compaction (all files) 2024-11-21T21:14:29,185 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae951cc3e18af3209220d547b1b63947/C in TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:29,185 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/593633d8540048baa886a565bdd368ec, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/eae3b0e9ff0046e79dcdd787b380850b, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/6b65140ebaf2405ca0b387ff1340569b] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp, totalSize=36.4 K 2024-11-21T21:14:29,186 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 593633d8540048baa886a565bdd368ec, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732223665744 2024-11-21T21:14:29,187 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting eae3b0e9ff0046e79dcdd787b380850b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732223667997 2024-11-21T21:14:29,188 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b65140ebaf2405ca0b387ff1340569b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732223668179 2024-11-21T21:14:29,202 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae951cc3e18af3209220d547b1b63947#C#compaction#59 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:14:29,202 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/f3311a3e16ee492ea9b1b29da97c6917 is 50, key is test_row_0/C:col10/1732223668512/Put/seqid=0 2024-11-21T21:14:29,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741896_1072 (size=12949) 2024-11-21T21:14:29,221 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/f3311a3e16ee492ea9b1b29da97c6917 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/f3311a3e16ee492ea9b1b29da97c6917 2024-11-21T21:14:29,237 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae951cc3e18af3209220d547b1b63947/C of ae951cc3e18af3209220d547b1b63947 into f3311a3e16ee492ea9b1b29da97c6917(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:14:29,237 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:29,240 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., storeName=ae951cc3e18af3209220d547b1b63947/C, priority=13, startTime=1732223669111; duration=0sec 2024-11-21T21:14:29,240 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:29,240 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae951cc3e18af3209220d547b1b63947:C 2024-11-21T21:14:29,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T21:14:29,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:29,244 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing ae951cc3e18af3209220d547b1b63947 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-21T21:14:29,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=A 2024-11-21T21:14:29,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:14:29,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=B 2024-11-21T21:14:29,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:14:29,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=C 2024-11-21T21:14:29,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:14:29,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/2914a3db94654c3890a542f5450d6cc5 is 50, key is test_row_0/A:col10/1732223668645/Put/seqid=0 2024-11-21T21:14:29,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741897_1073 (size=12301) 2024-11-21T21:14:29,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on ae951cc3e18af3209220d547b1b63947 2024-11-21T21:14:29,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:29,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:29,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223729317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:29,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:29,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:29,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:29,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:29,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223729317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:29,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:29,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:29,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:29,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:29,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223729317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:29,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:29,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223729319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:29,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:29,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:29,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:29,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:29,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:29,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:29,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:29,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223729324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:29,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:29,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:29,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:29,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T21:14:29,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:29,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223729431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:29,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:29,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:29,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:29,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:29,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223729431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:29,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:29,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:29,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:29,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:29,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223729440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:29,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:29,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223729440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:29,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:29,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:29,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:29,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:29,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:29,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223729440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:29,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:29,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:29,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:29,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:29,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:29,557 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/1089d7e37d2a4c3885839c69d7f760f6 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/1089d7e37d2a4c3885839c69d7f760f6 2024-11-21T21:14:29,564 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae951cc3e18af3209220d547b1b63947/B of ae951cc3e18af3209220d547b1b63947 into 1089d7e37d2a4c3885839c69d7f760f6(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:14:29,565 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:29,565 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., storeName=ae951cc3e18af3209220d547b1b63947/B, priority=13, startTime=1732223669110; duration=0sec 2024-11-21T21:14:29,565 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:29,565 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae951cc3e18af3209220d547b1b63947:B 2024-11-21T21:14:29,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:29,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223729640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:29,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:29,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:29,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:29,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:29,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223729640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:29,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:29,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:29,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:29,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:29,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223729651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:29,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:29,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:29,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:29,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223729651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:29,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:29,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:29,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:29,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:29,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:29,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223729651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:29,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:29,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:29,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:29,663 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/2914a3db94654c3890a542f5450d6cc5 2024-11-21T21:14:29,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/b5c8d3fbe6834fb49a4e8853390b5baa is 50, key is test_row_0/B:col10/1732223668645/Put/seqid=0 2024-11-21T21:14:29,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741898_1074 (size=12301) 2024-11-21T21:14:29,694 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/b5c8d3fbe6834fb49a4e8853390b5baa 2024-11-21T21:14:29,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/709d40e8073f4bd6a4a202a740a41b18 is 50, key is test_row_0/C:col10/1732223668645/Put/seqid=0 2024-11-21T21:14:29,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741899_1075 (size=12301) 2024-11-21T21:14:29,713 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/709d40e8073f4bd6a4a202a740a41b18 2024-11-21T21:14:29,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/2914a3db94654c3890a542f5450d6cc5 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/2914a3db94654c3890a542f5450d6cc5 2024-11-21T21:14:29,724 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/2914a3db94654c3890a542f5450d6cc5, entries=150, sequenceid=318, filesize=12.0 K 2024-11-21T21:14:29,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/b5c8d3fbe6834fb49a4e8853390b5baa as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/b5c8d3fbe6834fb49a4e8853390b5baa 2024-11-21T21:14:29,739 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/b5c8d3fbe6834fb49a4e8853390b5baa, entries=150, sequenceid=318, filesize=12.0 K 2024-11-21T21:14:29,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/709d40e8073f4bd6a4a202a740a41b18 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/709d40e8073f4bd6a4a202a740a41b18 2024-11-21T21:14:29,748 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/709d40e8073f4bd6a4a202a740a41b18, entries=150, sequenceid=318, filesize=12.0 K 2024-11-21T21:14:29,750 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for ae951cc3e18af3209220d547b1b63947 in 505ms, sequenceid=318, compaction requested=false 2024-11-21T21:14:29,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:29,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:29,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-21T21:14:29,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-21T21:14:29,758 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-21T21:14:29,758 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4540 sec 2024-11-21T21:14:29,766 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.4680 sec 2024-11-21T21:14:29,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on ae951cc3e18af3209220d547b1b63947 2024-11-21T21:14:29,952 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae951cc3e18af3209220d547b1b63947 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-21T21:14:29,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=A 2024-11-21T21:14:29,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:29,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=B 2024-11-21T21:14:29,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:29,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=C 2024-11-21T21:14:29,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:29,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/92a6c06c735640c0ba29fe99250d248f is 50, key is test_row_0/A:col10/1732223669315/Put/seqid=0 2024-11-21T21:14:29,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741900_1076 (size=14741) 2024-11-21T21:14:29,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:29,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223729986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:29,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:29,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:29,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:29,993 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1732223669975, max timestamp=1732223669990Num uniques 859; 2024-11-21T21:14:29,993 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1732223669975, max timestamp=1732223669990Num uniques 859; 2024-11-21T21:14:29,993 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-21T21:14:29,993 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1732223669975, max timestamp=1732223669990Num uniques 859; 2024-11-21T21:14:29,993 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-21T21:14:29,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-21T21:14:30,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:30,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223730011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:30,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:30,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:30,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:30,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:30,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223730012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:30,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:30,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:30,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:30,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:30,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223730018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:30,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:30,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:30,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223730019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:30,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:30,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:30,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:30,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:30,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:30,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:30,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223730100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:30,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:30,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:30,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:30,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:30,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:30,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223730131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:30,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223730131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:30,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:30,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:30,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:30,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:30,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223730131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:30,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:30,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:30,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:30,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:30,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:30,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:30,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:30,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223730140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:30,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:30,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:30,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:30,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:30,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223730311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:30,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:30,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:30,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:30,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:30,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223730340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:30,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:30,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:30,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:30,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:30,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223730341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:30,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:30,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:30,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:30,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:30,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223730341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:30,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:30,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:30,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:30,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:30,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223730360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:30,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:30,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:30,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:30,379 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/92a6c06c735640c0ba29fe99250d248f 2024-11-21T21:14:30,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/33fd4ef03c224d8cb8184abf03792943 is 50, key is test_row_0/B:col10/1732223669315/Put/seqid=0 2024-11-21T21:14:30,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741901_1077 (size=12301) 2024-11-21T21:14:30,393 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/33fd4ef03c224d8cb8184abf03792943 2024-11-21T21:14:30,409 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/c7d797920e5f44688edee669f51c633a is 50, key is test_row_0/C:col10/1732223669315/Put/seqid=0 2024-11-21T21:14:30,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741902_1078 (size=12301) 2024-11-21T21:14:30,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T21:14:30,440 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:14:30,441 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:14:30,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-21T21:14:30,443 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:14:30,444 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:14:30,444 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:14:30,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T21:14:30,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T21:14:30,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-21T21:14:30,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:30,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:30,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:30,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:30,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:30,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:30,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:30,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223730620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:30,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:30,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:30,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:30,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:30,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223730650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:30,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:30,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:30,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:30,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:30,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223730650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:30,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:30,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:30,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:30,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:30,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223730650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:30,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:30,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:30,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:30,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:30,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223730671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:30,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:30,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:30,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:30,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-21T21:14:30,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:30,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:30,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:30,751 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:30,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:30,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:30,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T21:14:30,814 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/c7d797920e5f44688edee669f51c633a 2024-11-21T21:14:30,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/92a6c06c735640c0ba29fe99250d248f as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/92a6c06c735640c0ba29fe99250d248f 2024-11-21T21:14:30,828 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/92a6c06c735640c0ba29fe99250d248f, entries=200, sequenceid=330, filesize=14.4 K 2024-11-21T21:14:30,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/33fd4ef03c224d8cb8184abf03792943 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/33fd4ef03c224d8cb8184abf03792943 2024-11-21T21:14:30,834 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/33fd4ef03c224d8cb8184abf03792943, entries=150, sequenceid=330, filesize=12.0 K 2024-11-21T21:14:30,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/c7d797920e5f44688edee669f51c633a as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/c7d797920e5f44688edee669f51c633a 2024-11-21T21:14:30,841 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/c7d797920e5f44688edee669f51c633a, entries=150, sequenceid=330, filesize=12.0 K 2024-11-21T21:14:30,842 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=140.89 KB/144270 for ae951cc3e18af3209220d547b1b63947 in 890ms, sequenceid=330, compaction requested=true 2024-11-21T21:14:30,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:30,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae951cc3e18af3209220d547b1b63947:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:14:30,843 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:14:30,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:30,843 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:14:30,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae951cc3e18af3209220d547b1b63947:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:14:30,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:30,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae951cc3e18af3209220d547b1b63947:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:14:30,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:14:30,844 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:14:30,844 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:14:30,844 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): ae951cc3e18af3209220d547b1b63947/B is initiating minor compaction (all files) 2024-11-21T21:14:30,844 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): ae951cc3e18af3209220d547b1b63947/A is initiating minor compaction (all files) 2024-11-21T21:14:30,844 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae951cc3e18af3209220d547b1b63947/A in TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:30,844 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae951cc3e18af3209220d547b1b63947/B in TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:30,845 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/42032c7c555746a69a461e5e55f85f0a, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/2914a3db94654c3890a542f5450d6cc5, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/92a6c06c735640c0ba29fe99250d248f] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp, totalSize=39.1 K 2024-11-21T21:14:30,845 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/1089d7e37d2a4c3885839c69d7f760f6, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/b5c8d3fbe6834fb49a4e8853390b5baa, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/33fd4ef03c224d8cb8184abf03792943] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp, totalSize=36.7 K 2024-11-21T21:14:30,845 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42032c7c555746a69a461e5e55f85f0a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732223668179 2024-11-21T21:14:30,845 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 1089d7e37d2a4c3885839c69d7f760f6, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732223668179 2024-11-21T21:14:30,845 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting b5c8d3fbe6834fb49a4e8853390b5baa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1732223668645 2024-11-21T21:14:30,845 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2914a3db94654c3890a542f5450d6cc5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1732223668645 2024-11-21T21:14:30,846 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92a6c06c735640c0ba29fe99250d248f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732223669315 2024-11-21T21:14:30,846 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 33fd4ef03c224d8cb8184abf03792943, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732223669315 2024-11-21T21:14:30,855 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae951cc3e18af3209220d547b1b63947#B#compaction#66 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:14:30,856 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/8a73c26acbd54c84a83b16db0ed60451 is 50, key is test_row_0/B:col10/1732223669315/Put/seqid=0 2024-11-21T21:14:30,858 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae951cc3e18af3209220d547b1b63947#A#compaction#67 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:14:30,859 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/72070a5fc8b04ffeba7a4dd35a990682 is 50, key is test_row_0/A:col10/1732223669315/Put/seqid=0 2024-11-21T21:14:30,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741903_1079 (size=13051) 2024-11-21T21:14:30,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741904_1080 (size=13051) 2024-11-21T21:14:30,878 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/72070a5fc8b04ffeba7a4dd35a990682 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/72070a5fc8b04ffeba7a4dd35a990682 2024-11-21T21:14:30,887 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae951cc3e18af3209220d547b1b63947/A of ae951cc3e18af3209220d547b1b63947 into 72070a5fc8b04ffeba7a4dd35a990682(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:14:30,887 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:30,887 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., storeName=ae951cc3e18af3209220d547b1b63947/A, priority=13, startTime=1732223670843; duration=0sec 2024-11-21T21:14:30,888 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:14:30,888 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae951cc3e18af3209220d547b1b63947:A 2024-11-21T21:14:30,888 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:14:30,889 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:14:30,890 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): ae951cc3e18af3209220d547b1b63947/C is initiating minor compaction (all files) 2024-11-21T21:14:30,890 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae951cc3e18af3209220d547b1b63947/C in TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:30,890 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/f3311a3e16ee492ea9b1b29da97c6917, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/709d40e8073f4bd6a4a202a740a41b18, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/c7d797920e5f44688edee669f51c633a] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp, totalSize=36.7 K 2024-11-21T21:14:30,890 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3311a3e16ee492ea9b1b29da97c6917, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732223668179 2024-11-21T21:14:30,891 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 709d40e8073f4bd6a4a202a740a41b18, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1732223668645 2024-11-21T21:14:30,891 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7d797920e5f44688edee669f51c633a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732223669315 2024-11-21T21:14:30,900 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae951cc3e18af3209220d547b1b63947#C#compaction#68 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:14:30,901 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/5c738bb0c8254f74b0d111462fa7c013 is 50, key is test_row_0/C:col10/1732223669315/Put/seqid=0 2024-11-21T21:14:30,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-21T21:14:30,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:30,905 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing ae951cc3e18af3209220d547b1b63947 3/3 column families, dataSize=140.89 KB heapSize=381.47 KB 2024-11-21T21:14:30,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=A 2024-11-21T21:14:30,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:14:30,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=B 2024-11-21T21:14:30,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:14:30,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=C 2024-11-21T21:14:30,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:14:30,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/26aa18e0409149f3a5ba94de8671c301 is 50, key is test_row_0/A:col10/1732223670016/Put/seqid=0 2024-11-21T21:14:30,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741905_1081 (size=13051) 2024-11-21T21:14:30,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741906_1082 (size=12301) 2024-11-21T21:14:30,931 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/26aa18e0409149f3a5ba94de8671c301 2024-11-21T21:14:30,932 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/5c738bb0c8254f74b0d111462fa7c013 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/5c738bb0c8254f74b0d111462fa7c013 2024-11-21T21:14:30,941 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae951cc3e18af3209220d547b1b63947/C of ae951cc3e18af3209220d547b1b63947 into 5c738bb0c8254f74b0d111462fa7c013(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:14:30,942 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:30,942 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., storeName=ae951cc3e18af3209220d547b1b63947/C, priority=13, startTime=1732223670843; duration=0sec 2024-11-21T21:14:30,942 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:30,942 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae951cc3e18af3209220d547b1b63947:C 2024-11-21T21:14:30,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/5d81925eaedf43f190b119986a6332ec is 50, key is test_row_0/B:col10/1732223670016/Put/seqid=0 2024-11-21T21:14:30,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741907_1083 (size=12301) 2024-11-21T21:14:31,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T21:14:31,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on ae951cc3e18af3209220d547b1b63947 2024-11-21T21:14:31,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:31,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:31,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223731161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:31,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:31,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:31,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:31,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:31,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223731166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:31,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:31,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:31,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:31,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:31,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:31,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223731167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:31,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223731171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:31,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:31,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:31,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:31,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:31,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:31,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:31,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:31,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223731190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:31,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:31,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:31,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:31,269 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/8a73c26acbd54c84a83b16db0ed60451 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/8a73c26acbd54c84a83b16db0ed60451 2024-11-21T21:14:31,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:31,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223731270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:31,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:31,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:31,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:31,278 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae951cc3e18af3209220d547b1b63947/B of ae951cc3e18af3209220d547b1b63947 into 8a73c26acbd54c84a83b16db0ed60451(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:14:31,278 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:31,278 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., storeName=ae951cc3e18af3209220d547b1b63947/B, priority=13, startTime=1732223670843; duration=0sec 2024-11-21T21:14:31,278 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:31,278 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae951cc3e18af3209220d547b1b63947:B 2024-11-21T21:14:31,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:31,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223731280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:31,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:31,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:31,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:31,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:31,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223731281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:31,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:31,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:31,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:31,348 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/5d81925eaedf43f190b119986a6332ec 2024-11-21T21:14:31,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/9fc1bc30b191424295ecc366b370a34a is 50, key is test_row_0/C:col10/1732223670016/Put/seqid=0 2024-11-21T21:14:31,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741908_1084 (size=12301) 2024-11-21T21:14:31,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:31,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223731480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:31,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:31,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:31,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:31,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:31,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223731491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:31,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:31,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223731491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:31,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:31,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:31,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:31,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:31,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:31,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:31,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T21:14:31,762 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/9fc1bc30b191424295ecc366b370a34a 2024-11-21T21:14:31,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/26aa18e0409149f3a5ba94de8671c301 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/26aa18e0409149f3a5ba94de8671c301 2024-11-21T21:14:31,775 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/26aa18e0409149f3a5ba94de8671c301, entries=150, sequenceid=364, filesize=12.0 K 2024-11-21T21:14:31,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/5d81925eaedf43f190b119986a6332ec as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/5d81925eaedf43f190b119986a6332ec 2024-11-21T21:14:31,782 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/5d81925eaedf43f190b119986a6332ec, entries=150, sequenceid=364, filesize=12.0 K 2024-11-21T21:14:31,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/9fc1bc30b191424295ecc366b370a34a as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/9fc1bc30b191424295ecc366b370a34a 2024-11-21T21:14:31,789 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/9fc1bc30b191424295ecc366b370a34a, entries=150, sequenceid=364, filesize=12.0 K 2024-11-21T21:14:31,791 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~381.42 KB/390576, currentSize=53.67 KB/54960 for ae951cc3e18af3209220d547b1b63947 in 886ms, sequenceid=364, compaction requested=false 2024-11-21T21:14:31,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:31,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:31,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-21T21:14:31,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-21T21:14:31,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on ae951cc3e18af3209220d547b1b63947 2024-11-21T21:14:31,794 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae951cc3e18af3209220d547b1b63947 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-21T21:14:31,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=A 2024-11-21T21:14:31,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:31,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=B 2024-11-21T21:14:31,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:31,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=C 2024-11-21T21:14:31,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:31,796 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-21T21:14:31,796 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3490 sec 2024-11-21T21:14:31,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/7a8088da3899455cb2ebaeb59e4a0286 is 50, key is test_row_0/A:col10/1732223671794/Put/seqid=0 2024-11-21T21:14:31,804 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.3580 sec 2024-11-21T21:14:31,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741909_1085 (size=14741) 2024-11-21T21:14:31,815 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/7a8088da3899455cb2ebaeb59e4a0286 2024-11-21T21:14:31,828 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/fb0bfff8841b40e39ef57b9d09211b75 is 50, key is test_row_0/B:col10/1732223671794/Put/seqid=0 2024-11-21T21:14:31,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741910_1086 (size=12301) 2024-11-21T21:14:31,841 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732223671815, max timestamp=1732223671837Num uniques 859; 2024-11-21T21:14:31,841 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732223671815, max timestamp=1732223671837Num uniques 859; 2024-11-21T21:14:31,841 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-21T21:14:31,841 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-21T21:14:31,843 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732223671815, max timestamp=1732223671837Num uniques 859; 2024-11-21T21:14:31,844 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-21T21:14:31,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:31,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223731897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:31,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:31,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:31,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:31,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:31,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223731898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:31,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:31,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:31,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:31,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:31,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223731899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:31,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:31,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:31,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:32,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:32,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223732010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:32,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:32,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:32,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223732010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:32,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:32,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:32,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:32,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:32,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223732010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:32,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:32,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:32,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:32,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:32,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:32,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:32,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223732180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:32,187 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,41213,1732223653469 for 'test_row_0' in ae951cc3e18af3209220d547b1b63947 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:14:32,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:32,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:32,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:32,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:32,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223732200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:32,201 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,41213,1732223653469 for 'test_row_2' in ae951cc3e18af3209220d547b1b63947 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:14:32,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:32,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:32,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:32,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:32,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223732220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:32,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:32,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:32,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223732220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:32,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:32,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:32,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:32,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:32,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:32,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:32,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223732221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:32,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:32,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:32,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:32,234 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/fb0bfff8841b40e39ef57b9d09211b75 2024-11-21T21:14:32,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/015ab247541945c4804d2890f2e1848c is 50, key is test_row_0/C:col10/1732223671794/Put/seqid=0 2024-11-21T21:14:32,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741911_1087 (size=12301) 2024-11-21T21:14:32,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/015ab247541945c4804d2890f2e1848c 2024-11-21T21:14:32,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/7a8088da3899455cb2ebaeb59e4a0286 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/7a8088da3899455cb2ebaeb59e4a0286 2024-11-21T21:14:32,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/7a8088da3899455cb2ebaeb59e4a0286, entries=200, sequenceid=378, filesize=14.4 K 2024-11-21T21:14:32,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/fb0bfff8841b40e39ef57b9d09211b75 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/fb0bfff8841b40e39ef57b9d09211b75 2024-11-21T21:14:32,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/fb0bfff8841b40e39ef57b9d09211b75, entries=150, sequenceid=378, filesize=12.0 K 2024-11-21T21:14:32,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/015ab247541945c4804d2890f2e1848c as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/015ab247541945c4804d2890f2e1848c 2024-11-21T21:14:32,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/015ab247541945c4804d2890f2e1848c, entries=150, sequenceid=378, filesize=12.0 K 2024-11-21T21:14:32,276 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for ae951cc3e18af3209220d547b1b63947 in 482ms, sequenceid=378, compaction requested=true 2024-11-21T21:14:32,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:32,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae951cc3e18af3209220d547b1b63947:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:14:32,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:32,276 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:14:32,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae951cc3e18af3209220d547b1b63947:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:14:32,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:32,276 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:14:32,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae951cc3e18af3209220d547b1b63947:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:14:32,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:14:32,277 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:14:32,277 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:14:32,278 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): ae951cc3e18af3209220d547b1b63947/B is initiating minor compaction (all files) 2024-11-21T21:14:32,278 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): ae951cc3e18af3209220d547b1b63947/A is initiating minor compaction (all files) 2024-11-21T21:14:32,278 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae951cc3e18af3209220d547b1b63947/B in TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:32,278 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae951cc3e18af3209220d547b1b63947/A in TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:32,278 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/8a73c26acbd54c84a83b16db0ed60451, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/5d81925eaedf43f190b119986a6332ec, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/fb0bfff8841b40e39ef57b9d09211b75] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp, totalSize=36.8 K 2024-11-21T21:14:32,278 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/72070a5fc8b04ffeba7a4dd35a990682, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/26aa18e0409149f3a5ba94de8671c301, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/7a8088da3899455cb2ebaeb59e4a0286] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp, totalSize=39.2 K 2024-11-21T21:14:32,278 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72070a5fc8b04ffeba7a4dd35a990682, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732223669315 2024-11-21T21:14:32,279 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a73c26acbd54c84a83b16db0ed60451, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732223669315 2024-11-21T21:14:32,279 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26aa18e0409149f3a5ba94de8671c301, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1732223669997 2024-11-21T21:14:32,279 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d81925eaedf43f190b119986a6332ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1732223669997 2024-11-21T21:14:32,279 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting fb0bfff8841b40e39ef57b9d09211b75, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732223671138 2024-11-21T21:14:32,280 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a8088da3899455cb2ebaeb59e4a0286, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732223671138 2024-11-21T21:14:32,294 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae951cc3e18af3209220d547b1b63947#B#compaction#76 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:14:32,294 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae951cc3e18af3209220d547b1b63947#A#compaction#75 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:14:32,294 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/1a04fedf6edf4c11afdbefcffcf07846 is 50, key is test_row_0/B:col10/1732223671794/Put/seqid=0 2024-11-21T21:14:32,294 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/f44ffca992914049bd44a73afcd24768 is 50, key is test_row_0/A:col10/1732223671794/Put/seqid=0 2024-11-21T21:14:32,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741913_1089 (size=13153) 2024-11-21T21:14:32,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741912_1088 (size=13153) 2024-11-21T21:14:32,311 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/f44ffca992914049bd44a73afcd24768 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/f44ffca992914049bd44a73afcd24768 2024-11-21T21:14:32,320 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae951cc3e18af3209220d547b1b63947/A of ae951cc3e18af3209220d547b1b63947 into f44ffca992914049bd44a73afcd24768(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:14:32,321 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:32,321 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., storeName=ae951cc3e18af3209220d547b1b63947/A, priority=13, startTime=1732223672276; duration=0sec 2024-11-21T21:14:32,321 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:14:32,321 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae951cc3e18af3209220d547b1b63947:A 2024-11-21T21:14:32,321 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:14:32,322 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:14:32,322 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): ae951cc3e18af3209220d547b1b63947/C is initiating minor compaction (all files) 2024-11-21T21:14:32,322 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae951cc3e18af3209220d547b1b63947/C in TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:32,323 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/5c738bb0c8254f74b0d111462fa7c013, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/9fc1bc30b191424295ecc366b370a34a, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/015ab247541945c4804d2890f2e1848c] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp, totalSize=36.8 K 2024-11-21T21:14:32,323 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c738bb0c8254f74b0d111462fa7c013, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732223669315 2024-11-21T21:14:32,323 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fc1bc30b191424295ecc366b370a34a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1732223669997 2024-11-21T21:14:32,324 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 015ab247541945c4804d2890f2e1848c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732223671138 2024-11-21T21:14:32,333 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae951cc3e18af3209220d547b1b63947#C#compaction#77 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:14:32,334 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/3f936b85a7694f2bbee8a638c05fb4c9 is 50, key is test_row_0/C:col10/1732223671794/Put/seqid=0 2024-11-21T21:14:32,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741914_1090 (size=13153) 2024-11-21T21:14:32,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on ae951cc3e18af3209220d547b1b63947 2024-11-21T21:14:32,534 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae951cc3e18af3209220d547b1b63947 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-11-21T21:14:32,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=A 2024-11-21T21:14:32,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:14:32,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=B 2024-11-21T21:14:32,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:14:32,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=C 2024-11-21T21:14:32,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:14:32,541 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/e146707210234ffca6d269ce49d7f4f7 is 50, key is test_row_0/A:col10/1732223672533/Put/seqid=0 2024-11-21T21:14:32,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741915_1091 (size=14741) 2024-11-21T21:14:32,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:32,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223732542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:32,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:32,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:32,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:32,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:32,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223732543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:32,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:32,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:32,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:32,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:32,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223732553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:32,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:32,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:32,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:32,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T21:14:32,610 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:14:32,611 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:14:32,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-21T21:14:32,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T21:14:32,613 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:14:32,613 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:14:32,614 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:14:32,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:32,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:32,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223732660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:32,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223732660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:32,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:32,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:32,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:32,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223732661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:32,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:32,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:32,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:32,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:32,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:32,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:32,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:32,705 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/1a04fedf6edf4c11afdbefcffcf07846 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/1a04fedf6edf4c11afdbefcffcf07846 2024-11-21T21:14:32,711 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae951cc3e18af3209220d547b1b63947/B of ae951cc3e18af3209220d547b1b63947 into 1a04fedf6edf4c11afdbefcffcf07846(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:14:32,711 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:32,711 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., storeName=ae951cc3e18af3209220d547b1b63947/B, priority=13, startTime=1732223672276; duration=0sec 2024-11-21T21:14:32,712 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:32,712 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae951cc3e18af3209220d547b1b63947:B 2024-11-21T21:14:32,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T21:14:32,749 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/3f936b85a7694f2bbee8a638c05fb4c9 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/3f936b85a7694f2bbee8a638c05fb4c9 2024-11-21T21:14:32,754 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae951cc3e18af3209220d547b1b63947/C of ae951cc3e18af3209220d547b1b63947 into 3f936b85a7694f2bbee8a638c05fb4c9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:14:32,754 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:32,755 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., storeName=ae951cc3e18af3209220d547b1b63947/C, priority=13, startTime=1732223672276; duration=0sec 2024-11-21T21:14:32,755 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:32,755 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae951cc3e18af3209220d547b1b63947:C 2024-11-21T21:14:32,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-21T21:14:32,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:32,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:32,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:32,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:32,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:32,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:32,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:32,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223732871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:32,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:32,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:32,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223732871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:32,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:32,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:32,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:32,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:32,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:32,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:32,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223732871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:32,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:32,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:32,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:32,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-21T21:14:32,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:32,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:32,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:32,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:32,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:32,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:32,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T21:14:32,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/e146707210234ffca6d269ce49d7f4f7 2024-11-21T21:14:32,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/e3df436333b948bb9112ab1c9316f327 is 50, key is test_row_0/B:col10/1732223672533/Put/seqid=0 2024-11-21T21:14:32,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741916_1092 (size=12301) 2024-11-21T21:14:33,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-21T21:14:33,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:33,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:33,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:33,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:33,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:33,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:33,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:33,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:33,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223733180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:33,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223733180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:33,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:33,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:33,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:33,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223733180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:33,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:33,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:33,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:33,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:33,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:33,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:33,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:33,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-21T21:14:33,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:33,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:33,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:33,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:33,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:33,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:33,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T21:14:33,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/e3df436333b948bb9112ab1c9316f327 2024-11-21T21:14:33,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/f1a3656039c1464d8d5e8bf7afd0fbca is 50, key is test_row_0/C:col10/1732223672533/Put/seqid=0 2024-11-21T21:14:33,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741917_1093 (size=12301) 2024-11-21T21:14:33,381 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/f1a3656039c1464d8d5e8bf7afd0fbca 2024-11-21T21:14:33,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-21T21:14:33,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:33,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:33,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:33,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:33,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:33,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:33,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/e146707210234ffca6d269ce49d7f4f7 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/e146707210234ffca6d269ce49d7f4f7 2024-11-21T21:14:33,395 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/e146707210234ffca6d269ce49d7f4f7, entries=200, sequenceid=414, filesize=14.4 K 2024-11-21T21:14:33,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/e3df436333b948bb9112ab1c9316f327 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/e3df436333b948bb9112ab1c9316f327 2024-11-21T21:14:33,404 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/e3df436333b948bb9112ab1c9316f327, entries=150, sequenceid=414, filesize=12.0 K 2024-11-21T21:14:33,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/f1a3656039c1464d8d5e8bf7afd0fbca as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/f1a3656039c1464d8d5e8bf7afd0fbca 2024-11-21T21:14:33,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/f1a3656039c1464d8d5e8bf7afd0fbca, entries=150, sequenceid=414, filesize=12.0 K 2024-11-21T21:14:33,414 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=46.96 KB/48090 for ae951cc3e18af3209220d547b1b63947 in 880ms, sequenceid=414, compaction requested=false 2024-11-21T21:14:33,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:33,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-21T21:14:33,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:33,539 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing ae951cc3e18af3209220d547b1b63947 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T21:14:33,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=A 2024-11-21T21:14:33,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:33,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=B 2024-11-21T21:14:33,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:33,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=C 2024-11-21T21:14:33,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:33,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/eeb709c8e18644adaf73f02a1bc6b096 is 50, key is test_row_0/A:col10/1732223672541/Put/seqid=0 2024-11-21T21:14:33,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741918_1094 (size=12301) 2024-11-21T21:14:33,565 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/eeb709c8e18644adaf73f02a1bc6b096 2024-11-21T21:14:33,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/0bc8d70ee6b9473cbdb431b482380b2c is 50, key is test_row_0/B:col10/1732223672541/Put/seqid=0 2024-11-21T21:14:33,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741919_1095 (size=12301) 2024-11-21T21:14:33,598 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/0bc8d70ee6b9473cbdb431b482380b2c 2024-11-21T21:14:33,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/7e6b40ff80374d858231eb23a895d193 is 50, key is test_row_0/C:col10/1732223672541/Put/seqid=0 2024-11-21T21:14:33,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741920_1096 (size=12301) 2024-11-21T21:14:33,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on ae951cc3e18af3209220d547b1b63947 2024-11-21T21:14:33,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:33,724 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732223673693, max timestamp=1732223673722Num uniques 859; 2024-11-21T21:14:33,724 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732223673693, max timestamp=1732223673722Num uniques 859; 2024-11-21T21:14:33,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732223673693, max timestamp=1732223673722Num uniques 859; 2024-11-21T21:14:33,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-21T21:14:33,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-21T21:14:33,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-21T21:14:33,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T21:14:33,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:33,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:33,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223733751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:33,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223733753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:33,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:33,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:33,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:33,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223733754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:33,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:33,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:33,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:33,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:33,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:33,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:33,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:33,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:33,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223733871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:33,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:33,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:33,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:33,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:33,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223733872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:33,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:33,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:33,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:33,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:33,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223733872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:33,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:33,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:33,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:34,021 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/7e6b40ff80374d858231eb23a895d193 2024-11-21T21:14:34,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/eeb709c8e18644adaf73f02a1bc6b096 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/eeb709c8e18644adaf73f02a1bc6b096 2024-11-21T21:14:34,037 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/eeb709c8e18644adaf73f02a1bc6b096, entries=150, sequenceid=426, filesize=12.0 K 2024-11-21T21:14:34,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/0bc8d70ee6b9473cbdb431b482380b2c as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/0bc8d70ee6b9473cbdb431b482380b2c 2024-11-21T21:14:34,044 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/0bc8d70ee6b9473cbdb431b482380b2c, entries=150, sequenceid=426, filesize=12.0 K 2024-11-21T21:14:34,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/7e6b40ff80374d858231eb23a895d193 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/7e6b40ff80374d858231eb23a895d193 2024-11-21T21:14:34,055 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/7e6b40ff80374d858231eb23a895d193, entries=150, sequenceid=426, filesize=12.0 K 2024-11-21T21:14:34,057 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for ae951cc3e18af3209220d547b1b63947 in 518ms, sequenceid=426, compaction requested=true 2024-11-21T21:14:34,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:34,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:34,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-21T21:14:34,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-21T21:14:34,061 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-21T21:14:34,061 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4450 sec 2024-11-21T21:14:34,063 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.4510 sec 2024-11-21T21:14:34,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on ae951cc3e18af3209220d547b1b63947 2024-11-21T21:14:34,083 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae951cc3e18af3209220d547b1b63947 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-11-21T21:14:34,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=A 2024-11-21T21:14:34,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:14:34,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=B 2024-11-21T21:14:34,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:14:34,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=C 2024-11-21T21:14:34,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:14:34,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/09284d4fd3204cd3be4347922948d1b9 is 50, key is test_row_0/A:col10/1732223673749/Put/seqid=0 2024-11-21T21:14:34,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741921_1097 (size=17181) 2024-11-21T21:14:34,098 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/09284d4fd3204cd3be4347922948d1b9 2024-11-21T21:14:34,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:34,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223734093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:34,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:34,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:34,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:34,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:34,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223734094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:34,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:34,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:34,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:34,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:34,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223734101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:34,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:34,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:34,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:34,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/66f2a9db07d241b4848a7055d2be8ccb is 50, key is test_row_0/B:col10/1732223673749/Put/seqid=0 2024-11-21T21:14:34,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741922_1098 (size=12301) 2024-11-21T21:14:34,118 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/66f2a9db07d241b4848a7055d2be8ccb 2024-11-21T21:14:34,132 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/aa3b58c115594c9ca62484e6a40b17c0 is 50, key is test_row_0/C:col10/1732223673749/Put/seqid=0 2024-11-21T21:14:34,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741923_1099 (size=12301) 2024-11-21T21:14:34,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/aa3b58c115594c9ca62484e6a40b17c0 2024-11-21T21:14:34,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/09284d4fd3204cd3be4347922948d1b9 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/09284d4fd3204cd3be4347922948d1b9 2024-11-21T21:14:34,161 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/09284d4fd3204cd3be4347922948d1b9, entries=250, sequenceid=463, filesize=16.8 K 2024-11-21T21:14:34,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/66f2a9db07d241b4848a7055d2be8ccb as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/66f2a9db07d241b4848a7055d2be8ccb 2024-11-21T21:14:34,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/66f2a9db07d241b4848a7055d2be8ccb, entries=150, sequenceid=463, filesize=12.0 K 2024-11-21T21:14:34,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/aa3b58c115594c9ca62484e6a40b17c0 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/aa3b58c115594c9ca62484e6a40b17c0 2024-11-21T21:14:34,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/aa3b58c115594c9ca62484e6a40b17c0, entries=150, sequenceid=463, filesize=12.0 K 2024-11-21T21:14:34,179 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~430.64 KB/440976, currentSize=33.54 KB/34350 for ae951cc3e18af3209220d547b1b63947 in 96ms, sequenceid=463, compaction requested=true 2024-11-21T21:14:34,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:34,179 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:14:34,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae951cc3e18af3209220d547b1b63947:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:14:34,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:34,180 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:14:34,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae951cc3e18af3209220d547b1b63947:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:14:34,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:34,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae951cc3e18af3209220d547b1b63947:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:14:34,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:14:34,181 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57376 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:14:34,182 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): ae951cc3e18af3209220d547b1b63947/A is initiating minor compaction (all files) 2024-11-21T21:14:34,182 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae951cc3e18af3209220d547b1b63947/A in TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:34,182 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/f44ffca992914049bd44a73afcd24768, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/e146707210234ffca6d269ce49d7f4f7, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/eeb709c8e18644adaf73f02a1bc6b096, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/09284d4fd3204cd3be4347922948d1b9] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp, totalSize=56.0 K 2024-11-21T21:14:34,182 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:14:34,182 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): ae951cc3e18af3209220d547b1b63947/B is initiating minor compaction (all files) 2024-11-21T21:14:34,183 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae951cc3e18af3209220d547b1b63947/B in TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:34,183 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/1a04fedf6edf4c11afdbefcffcf07846, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/e3df436333b948bb9112ab1c9316f327, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/0bc8d70ee6b9473cbdb431b482380b2c, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/66f2a9db07d241b4848a7055d2be8ccb] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp, totalSize=48.9 K 2024-11-21T21:14:34,183 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting f44ffca992914049bd44a73afcd24768, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732223671138 2024-11-21T21:14:34,183 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a04fedf6edf4c11afdbefcffcf07846, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732223671138 2024-11-21T21:14:34,184 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting e146707210234ffca6d269ce49d7f4f7, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1732223671896 2024-11-21T21:14:34,184 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting e3df436333b948bb9112ab1c9316f327, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1732223671897 2024-11-21T21:14:34,185 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting eeb709c8e18644adaf73f02a1bc6b096, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1732223672539 2024-11-21T21:14:34,185 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bc8d70ee6b9473cbdb431b482380b2c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1732223672539 2024-11-21T21:14:34,185 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09284d4fd3204cd3be4347922948d1b9, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1732223673733 2024-11-21T21:14:34,187 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 66f2a9db07d241b4848a7055d2be8ccb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1732223673733 2024-11-21T21:14:34,202 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae951cc3e18af3209220d547b1b63947#A#compaction#87 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:14:34,202 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/c4c56e395c6f4a6d862a2442ff76666e is 50, key is test_row_0/A:col10/1732223673749/Put/seqid=0 2024-11-21T21:14:34,203 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae951cc3e18af3209220d547b1b63947#B#compaction#88 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:14:34,203 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/2e07dd17bdcc4ac6bfe7a5341378c1ce is 50, key is test_row_0/B:col10/1732223673749/Put/seqid=0 2024-11-21T21:14:34,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on ae951cc3e18af3209220d547b1b63947 2024-11-21T21:14:34,210 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae951cc3e18af3209220d547b1b63947 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T21:14:34,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=A 2024-11-21T21:14:34,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:34,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=B 2024-11-21T21:14:34,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:34,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=C 2024-11-21T21:14:34,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:34,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741924_1100 (size=13289) 2024-11-21T21:14:34,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/6a7162851c334e4dbf2061e204d72b04 is 50, key is test_row_0/A:col10/1732223674206/Put/seqid=0 2024-11-21T21:14:34,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741925_1101 (size=13289) 2024-11-21T21:14:34,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741926_1102 (size=14741) 2024-11-21T21:14:34,246 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732223674224, max timestamp=1732223674243Num uniques 859; 2024-11-21T21:14:34,246 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732223674224, max timestamp=1732223674243Num uniques 859; 2024-11-21T21:14:34,246 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-21T21:14:34,246 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-21T21:14:34,249 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732223674224, max timestamp=1732223674243Num uniques 859; 2024-11-21T21:14:34,249 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-21T21:14:34,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:34,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223734264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:34,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:34,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:34,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:34,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:34,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223734264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:34,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:34,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:34,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:34,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:34,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223734266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:34,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:34,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:34,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:34,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:34,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:34,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223734272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:34,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223734271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:34,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:34,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:34,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:34,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:34,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:34,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:34,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:34,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223734381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:34,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:34,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:34,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:34,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:34,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223734381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:34,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:34,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:34,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:34,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:34,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223734381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:34,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:34,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:34,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:34,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:34,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223734390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:34,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:34,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:34,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:34,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:34,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223734390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:34,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:34,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:34,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:34,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:34,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223734590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:34,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:34,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:34,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:34,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:34,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223734590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:34,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:34,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223734590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:34,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:34,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:34,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:34,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:34,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:34,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:34,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:34,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223734600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:34,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:34,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:34,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:34,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:34,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223734600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:34,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:34,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:34,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:34,626 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/c4c56e395c6f4a6d862a2442ff76666e as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/c4c56e395c6f4a6d862a2442ff76666e 2024-11-21T21:14:34,626 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/6a7162851c334e4dbf2061e204d72b04 2024-11-21T21:14:34,635 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/2e07dd17bdcc4ac6bfe7a5341378c1ce as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/2e07dd17bdcc4ac6bfe7a5341378c1ce 2024-11-21T21:14:34,639 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae951cc3e18af3209220d547b1b63947/A of ae951cc3e18af3209220d547b1b63947 into c4c56e395c6f4a6d862a2442ff76666e(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:14:34,639 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:34,639 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., storeName=ae951cc3e18af3209220d547b1b63947/A, priority=12, startTime=1732223674179; duration=0sec 2024-11-21T21:14:34,639 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:14:34,640 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae951cc3e18af3209220d547b1b63947:A 2024-11-21T21:14:34,640 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:14:34,643 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:14:34,643 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): ae951cc3e18af3209220d547b1b63947/C is initiating minor compaction (all files) 2024-11-21T21:14:34,643 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae951cc3e18af3209220d547b1b63947/C in TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:34,643 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/3f936b85a7694f2bbee8a638c05fb4c9, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/f1a3656039c1464d8d5e8bf7afd0fbca, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/7e6b40ff80374d858231eb23a895d193, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/aa3b58c115594c9ca62484e6a40b17c0] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp, totalSize=48.9 K 2024-11-21T21:14:34,644 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f936b85a7694f2bbee8a638c05fb4c9, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732223671138 2024-11-21T21:14:34,644 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1a3656039c1464d8d5e8bf7afd0fbca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1732223671897 2024-11-21T21:14:34,645 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e6b40ff80374d858231eb23a895d193, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1732223672539 2024-11-21T21:14:34,646 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa3b58c115594c9ca62484e6a40b17c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1732223673733 2024-11-21T21:14:34,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/2821679f2896432d97f10e5cc6b1216e is 50, key is test_row_0/B:col10/1732223674206/Put/seqid=0 2024-11-21T21:14:34,649 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae951cc3e18af3209220d547b1b63947/B of ae951cc3e18af3209220d547b1b63947 into 2e07dd17bdcc4ac6bfe7a5341378c1ce(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:14:34,649 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:34,649 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., storeName=ae951cc3e18af3209220d547b1b63947/B, priority=12, startTime=1732223674180; duration=0sec 2024-11-21T21:14:34,649 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:34,650 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae951cc3e18af3209220d547b1b63947:B 2024-11-21T21:14:34,658 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae951cc3e18af3209220d547b1b63947#C#compaction#91 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:14:34,659 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/dcf511c264544832b46491e510e9195b is 50, key is test_row_0/C:col10/1732223673749/Put/seqid=0 2024-11-21T21:14:34,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741927_1103 (size=12301) 2024-11-21T21:14:34,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741928_1104 (size=13289) 2024-11-21T21:14:34,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T21:14:34,760 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:14:34,761 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:14:34,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-21T21:14:34,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T21:14:34,763 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:14:34,764 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:14:34,764 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:14:34,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T21:14:34,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:34,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:34,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223734900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:34,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223734900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:34,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:34,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:34,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:34,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:34,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:34,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:34,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:34,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223734901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:34,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:34,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:34,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:34,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-21T21:14:34,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:34,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:34,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:34,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:34,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:34,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:34,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:34,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223734920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:34,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:34,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:34,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:34,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:34,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223734920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:34,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:34,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:34,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:35,067 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/2821679f2896432d97f10e5cc6b1216e 2024-11-21T21:14:35,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-21T21:14:35,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:35,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:35,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:35,071 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:35,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:35,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:35,075 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/dcf511c264544832b46491e510e9195b as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/dcf511c264544832b46491e510e9195b 2024-11-21T21:14:35,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T21:14:35,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/c09b9afebfc84071a2112c56ef66c823 is 50, key is test_row_0/C:col10/1732223674206/Put/seqid=0 2024-11-21T21:14:35,084 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae951cc3e18af3209220d547b1b63947/C of ae951cc3e18af3209220d547b1b63947 into dcf511c264544832b46491e510e9195b(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:14:35,084 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:35,084 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., storeName=ae951cc3e18af3209220d547b1b63947/C, priority=12, startTime=1732223674180; duration=0sec 2024-11-21T21:14:35,084 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:35,084 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae951cc3e18af3209220d547b1b63947:C 2024-11-21T21:14:35,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741929_1105 (size=12301) 2024-11-21T21:14:35,092 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/c09b9afebfc84071a2112c56ef66c823 2024-11-21T21:14:35,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/6a7162851c334e4dbf2061e204d72b04 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/6a7162851c334e4dbf2061e204d72b04 2024-11-21T21:14:35,104 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/6a7162851c334e4dbf2061e204d72b04, entries=200, sequenceid=473, filesize=14.4 K 2024-11-21T21:14:35,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/2821679f2896432d97f10e5cc6b1216e as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/2821679f2896432d97f10e5cc6b1216e 2024-11-21T21:14:35,112 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/2821679f2896432d97f10e5cc6b1216e, entries=150, sequenceid=473, filesize=12.0 K 2024-11-21T21:14:35,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/c09b9afebfc84071a2112c56ef66c823 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/c09b9afebfc84071a2112c56ef66c823 2024-11-21T21:14:35,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/c09b9afebfc84071a2112c56ef66c823, entries=150, sequenceid=473, filesize=12.0 K 2024-11-21T21:14:35,124 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for ae951cc3e18af3209220d547b1b63947 in 914ms, sequenceid=473, compaction requested=false 2024-11-21T21:14:35,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:35,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-21T21:14:35,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:35,243 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing ae951cc3e18af3209220d547b1b63947 3/3 column families, dataSize=140.89 KB heapSize=386.16 KB 2024-11-21T21:14:35,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=A 2024-11-21T21:14:35,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:14:35,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=B 2024-11-21T21:14:35,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:14:35,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=C 2024-11-21T21:14:35,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:14:35,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/c06580df8f724fb39bdcd8b4052c1f1c is 50, key is test_row_0/A:col10/1732223674269/Put/seqid=0 2024-11-21T21:14:35,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741930_1106 (size=12301) 2024-11-21T21:14:35,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T21:14:35,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on ae951cc3e18af3209220d547b1b63947 2024-11-21T21:14:35,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:35,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:35,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223735424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:35,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:35,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:35,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:35,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:35,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223735425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:35,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:35,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:35,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:35,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:35,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223735426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:35,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:35,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:35,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:35,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:35,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223735430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:35,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:35,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:35,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:35,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:35,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223735430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:35,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:35,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:35,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:35,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:35,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223735530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:35,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:35,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:35,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:35,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:35,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223735540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:35,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:35,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:35,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:35,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223735540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:35,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:35,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:35,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:35,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:35,656 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=510 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/c06580df8f724fb39bdcd8b4052c1f1c 2024-11-21T21:14:35,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/9008963171ec4606b2785e91fab6d690 is 50, key is test_row_0/B:col10/1732223674269/Put/seqid=0 2024-11-21T21:14:35,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741931_1107 (size=12301) 2024-11-21T21:14:35,680 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=510 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/9008963171ec4606b2785e91fab6d690 2024-11-21T21:14:35,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/aea7522982d94d51a0623c7d6029632a is 50, key is test_row_0/C:col10/1732223674269/Put/seqid=0 2024-11-21T21:14:35,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741932_1108 (size=12301) 2024-11-21T21:14:35,699 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=510 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/aea7522982d94d51a0623c7d6029632a 2024-11-21T21:14:35,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/c06580df8f724fb39bdcd8b4052c1f1c as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/c06580df8f724fb39bdcd8b4052c1f1c 2024-11-21T21:14:35,714 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/c06580df8f724fb39bdcd8b4052c1f1c, entries=150, sequenceid=510, filesize=12.0 K 2024-11-21T21:14:35,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/9008963171ec4606b2785e91fab6d690 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/9008963171ec4606b2785e91fab6d690 2024-11-21T21:14:35,721 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/9008963171ec4606b2785e91fab6d690, entries=150, sequenceid=510, filesize=12.0 K 2024-11-21T21:14:35,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/aea7522982d94d51a0623c7d6029632a as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/aea7522982d94d51a0623c7d6029632a 2024-11-21T21:14:35,732 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/aea7522982d94d51a0623c7d6029632a, entries=150, sequenceid=510, filesize=12.0 K 2024-11-21T21:14:35,734 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~386.11 KB/395376, currentSize=46.96 KB/48090 for ae951cc3e18af3209220d547b1b63947 in 490ms, sequenceid=510, compaction requested=true 2024-11-21T21:14:35,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:35,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:35,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-21T21:14:35,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-21T21:14:35,739 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-21T21:14:35,739 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 973 msec 2024-11-21T21:14:35,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on ae951cc3e18af3209220d547b1b63947 2024-11-21T21:14:35,742 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae951cc3e18af3209220d547b1b63947 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-21T21:14:35,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=A 2024-11-21T21:14:35,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:35,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=B 2024-11-21T21:14:35,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:35,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=C 2024-11-21T21:14:35,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:35,744 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 979 msec 2024-11-21T21:14:35,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/334e7d2d1d20497b89f46fefef4544a3 is 50, key is test_row_0/A:col10/1732223675424/Put/seqid=0 2024-11-21T21:14:35,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741933_1109 (size=14741) 2024-11-21T21:14:35,756 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/334e7d2d1d20497b89f46fefef4544a3 2024-11-21T21:14:35,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/07034f51312b409a8e347a88d64bb381 is 50, key is test_row_0/B:col10/1732223675424/Put/seqid=0 2024-11-21T21:14:35,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741934_1110 (size=12301) 2024-11-21T21:14:35,775 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/07034f51312b409a8e347a88d64bb381 2024-11-21T21:14:35,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/df5ee71ee1eb48ff948d0d5e0d8baba1 is 50, key is test_row_0/C:col10/1732223675424/Put/seqid=0 2024-11-21T21:14:35,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732223675772, max timestamp=1732223675787Num uniques 859; 2024-11-21T21:14:35,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T21:14:35,792 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732223675772, max timestamp=1732223675787Num uniques 859; 2024-11-21T21:14:35,792 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T21:14:35,794 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732223675772, max timestamp=1732223675787Num uniques 859; 2024-11-21T21:14:35,794 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T21:14:35,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741935_1111 (size=12301) 2024-11-21T21:14:35,798 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/df5ee71ee1eb48ff948d0d5e0d8baba1 2024-11-21T21:14:35,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/334e7d2d1d20497b89f46fefef4544a3 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/334e7d2d1d20497b89f46fefef4544a3 2024-11-21T21:14:35,815 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/334e7d2d1d20497b89f46fefef4544a3, entries=200, sequenceid=521, filesize=14.4 K 2024-11-21T21:14:35,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/07034f51312b409a8e347a88d64bb381 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/07034f51312b409a8e347a88d64bb381 2024-11-21T21:14:35,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/07034f51312b409a8e347a88d64bb381, entries=150, sequenceid=521, filesize=12.0 K 2024-11-21T21:14:35,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/df5ee71ee1eb48ff948d0d5e0d8baba1 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/df5ee71ee1eb48ff948d0d5e0d8baba1 2024-11-21T21:14:35,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/df5ee71ee1eb48ff948d0d5e0d8baba1, entries=150, sequenceid=521, filesize=12.0 K 2024-11-21T21:14:35,832 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=120.76 KB/123660 for ae951cc3e18af3209220d547b1b63947 in 91ms, sequenceid=521, compaction requested=true 2024-11-21T21:14:35,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:35,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae951cc3e18af3209220d547b1b63947:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:14:35,833 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:14:35,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:35,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae951cc3e18af3209220d547b1b63947:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:14:35,833 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:14:35,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:35,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae951cc3e18af3209220d547b1b63947:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:14:35,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:14:35,834 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55072 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:14:35,834 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): ae951cc3e18af3209220d547b1b63947/A is initiating minor compaction (all files) 2024-11-21T21:14:35,835 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae951cc3e18af3209220d547b1b63947/A in TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:35,835 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/c4c56e395c6f4a6d862a2442ff76666e, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/6a7162851c334e4dbf2061e204d72b04, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/c06580df8f724fb39bdcd8b4052c1f1c, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/334e7d2d1d20497b89f46fefef4544a3] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp, totalSize=53.8 K 2024-11-21T21:14:35,835 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:14:35,835 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): ae951cc3e18af3209220d547b1b63947/B is initiating minor compaction (all files) 2024-11-21T21:14:35,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on ae951cc3e18af3209220d547b1b63947 2024-11-21T21:14:35,835 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae951cc3e18af3209220d547b1b63947/B in TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:35,836 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/2e07dd17bdcc4ac6bfe7a5341378c1ce, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/2821679f2896432d97f10e5cc6b1216e, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/9008963171ec4606b2785e91fab6d690, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/07034f51312b409a8e347a88d64bb381] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp, totalSize=49.0 K 2024-11-21T21:14:35,836 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4c56e395c6f4a6d862a2442ff76666e, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1732223673733 2024-11-21T21:14:35,836 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae951cc3e18af3209220d547b1b63947 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-21T21:14:35,837 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e07dd17bdcc4ac6bfe7a5341378c1ce, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1732223673733 2024-11-21T21:14:35,837 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a7162851c334e4dbf2061e204d72b04, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1732223674085 2024-11-21T21:14:35,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=A 2024-11-21T21:14:35,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:14:35,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=B 2024-11-21T21:14:35,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:14:35,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=C 2024-11-21T21:14:35,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:14:35,838 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 2821679f2896432d97f10e5cc6b1216e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1732223674092 2024-11-21T21:14:35,838 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting c06580df8f724fb39bdcd8b4052c1f1c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=510, earliestPutTs=1732223674264 2024-11-21T21:14:35,839 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 334e7d2d1d20497b89f46fefef4544a3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1732223675414 2024-11-21T21:14:35,839 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 9008963171ec4606b2785e91fab6d690, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=510, earliestPutTs=1732223674264 2024-11-21T21:14:35,840 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 07034f51312b409a8e347a88d64bb381, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1732223675414 2024-11-21T21:14:35,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/070c4c3807fc4c4e8fe62a583171c90a is 50, key is test_row_0/A:col10/1732223675836/Put/seqid=0 2024-11-21T21:14:35,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:35,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223735849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:35,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:35,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:35,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:35,863 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae951cc3e18af3209220d547b1b63947#A#compaction#100 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:14:35,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:35,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223735857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:35,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:35,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741936_1112 (size=19621) 2024-11-21T21:14:35,865 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/607b9e2d69434a62a54085993ef14ed0 is 50, key is test_row_0/A:col10/1732223675424/Put/seqid=0 2024-11-21T21:14:35,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223735857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:35,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:35,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:35,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:35,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:35,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:35,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:35,866 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=557 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/070c4c3807fc4c4e8fe62a583171c90a 2024-11-21T21:14:35,871 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae951cc3e18af3209220d547b1b63947#B#compaction#101 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:14:35,872 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/7fe4b4f991294d0d9901abeb41c025af is 50, key is test_row_0/B:col10/1732223675424/Put/seqid=0 2024-11-21T21:14:35,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/fdd24163bb0e46f3ba8b607ec635d897 is 50, key is test_row_0/B:col10/1732223675836/Put/seqid=0 2024-11-21T21:14:35,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741937_1113 (size=13425) 2024-11-21T21:14:35,895 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/607b9e2d69434a62a54085993ef14ed0 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/607b9e2d69434a62a54085993ef14ed0 2024-11-21T21:14:35,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741938_1114 (size=13425) 2024-11-21T21:14:35,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T21:14:35,902 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:14:35,902 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:14:35,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-21T21:14:35,905 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae951cc3e18af3209220d547b1b63947/A of ae951cc3e18af3209220d547b1b63947 into 607b9e2d69434a62a54085993ef14ed0(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:14:35,905 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:35,905 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:14:35,905 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., storeName=ae951cc3e18af3209220d547b1b63947/A, priority=12, startTime=1732223675832; duration=0sec 2024-11-21T21:14:35,905 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:14:35,905 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae951cc3e18af3209220d547b1b63947:A 2024-11-21T21:14:35,905 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:14:35,906 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:14:35,906 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:14:35,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T21:14:35,915 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:14:35,915 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): ae951cc3e18af3209220d547b1b63947/C is initiating minor compaction (all files) 2024-11-21T21:14:35,915 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae951cc3e18af3209220d547b1b63947/C in TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:35,915 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/dcf511c264544832b46491e510e9195b, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/c09b9afebfc84071a2112c56ef66c823, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/aea7522982d94d51a0623c7d6029632a, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/df5ee71ee1eb48ff948d0d5e0d8baba1] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp, totalSize=49.0 K 2024-11-21T21:14:35,916 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting dcf511c264544832b46491e510e9195b, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1732223673733 2024-11-21T21:14:35,916 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting c09b9afebfc84071a2112c56ef66c823, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1732223674092 2024-11-21T21:14:35,918 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting aea7522982d94d51a0623c7d6029632a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=510, earliestPutTs=1732223674264 2024-11-21T21:14:35,918 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting df5ee71ee1eb48ff948d0d5e0d8baba1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1732223675414 2024-11-21T21:14:35,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741939_1115 (size=12301) 2024-11-21T21:14:35,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=557 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/fdd24163bb0e46f3ba8b607ec635d897 2024-11-21T21:14:35,940 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae951cc3e18af3209220d547b1b63947#C#compaction#103 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:14:35,941 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/0298a229fbfb48bcb9bcbe6d30fc22e4 is 50, key is test_row_0/C:col10/1732223675424/Put/seqid=0 2024-11-21T21:14:35,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/a2c15d8e080b42b1a32802ba3d0e176e is 50, key is test_row_0/C:col10/1732223675836/Put/seqid=0 2024-11-21T21:14:35,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741940_1116 (size=13425) 2024-11-21T21:14:35,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:35,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223735960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:35,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:35,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:35,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:35,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741941_1117 (size=12301) 2024-11-21T21:14:35,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:35,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223735970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:35,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:35,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:35,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:35,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:35,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223735970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:35,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:35,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:35,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:36,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T21:14:36,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-21T21:14:36,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:36,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:36,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:36,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:36,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:36,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:36,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:36,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223736170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:36,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:36,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:36,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:36,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:36,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223736180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:36,181 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:36,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:36,181 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:36,181 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:36,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223736180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:36,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:36,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:36,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:36,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-21T21:14:36,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:36,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:36,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:36,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:36,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:36,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:36,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T21:14:36,305 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/7fe4b4f991294d0d9901abeb41c025af as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/7fe4b4f991294d0d9901abeb41c025af 2024-11-21T21:14:36,312 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae951cc3e18af3209220d547b1b63947/B of ae951cc3e18af3209220d547b1b63947 into 7fe4b4f991294d0d9901abeb41c025af(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:14:36,312 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:36,312 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., storeName=ae951cc3e18af3209220d547b1b63947/B, priority=12, startTime=1732223675833; duration=0sec 2024-11-21T21:14:36,312 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:36,312 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae951cc3e18af3209220d547b1b63947:B 2024-11-21T21:14:36,359 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/0298a229fbfb48bcb9bcbe6d30fc22e4 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/0298a229fbfb48bcb9bcbe6d30fc22e4 2024-11-21T21:14:36,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=557 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/a2c15d8e080b42b1a32802ba3d0e176e 2024-11-21T21:14:36,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-21T21:14:36,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:36,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:36,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:36,366 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:36,366 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae951cc3e18af3209220d547b1b63947/C of ae951cc3e18af3209220d547b1b63947 into 0298a229fbfb48bcb9bcbe6d30fc22e4(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:14:36,366 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:36,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:36,367 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., storeName=ae951cc3e18af3209220d547b1b63947/C, priority=12, startTime=1732223675833; duration=0sec 2024-11-21T21:14:36,367 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:36,367 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae951cc3e18af3209220d547b1b63947:C 2024-11-21T21:14:36,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:36,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/070c4c3807fc4c4e8fe62a583171c90a as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/070c4c3807fc4c4e8fe62a583171c90a 2024-11-21T21:14:36,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/070c4c3807fc4c4e8fe62a583171c90a, entries=300, sequenceid=557, filesize=19.2 K 2024-11-21T21:14:36,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/fdd24163bb0e46f3ba8b607ec635d897 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/fdd24163bb0e46f3ba8b607ec635d897 2024-11-21T21:14:36,388 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/fdd24163bb0e46f3ba8b607ec635d897, entries=150, sequenceid=557, filesize=12.0 K 2024-11-21T21:14:36,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/a2c15d8e080b42b1a32802ba3d0e176e as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/a2c15d8e080b42b1a32802ba3d0e176e 2024-11-21T21:14:36,399 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/a2c15d8e080b42b1a32802ba3d0e176e, entries=150, sequenceid=557, filesize=12.0 K 2024-11-21T21:14:36,400 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for ae951cc3e18af3209220d547b1b63947 in 564ms, sequenceid=557, compaction requested=false 2024-11-21T21:14:36,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:36,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on ae951cc3e18af3209220d547b1b63947 2024-11-21T21:14:36,443 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae951cc3e18af3209220d547b1b63947 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-21T21:14:36,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=A 2024-11-21T21:14:36,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:36,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=B 2024-11-21T21:14:36,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:36,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=C 2024-11-21T21:14:36,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:36,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/fba1958d70f04821b2838b549030ae7c is 50, key is test_row_0/A:col10/1732223675838/Put/seqid=0 2024-11-21T21:14:36,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741942_1118 (size=17181) 2024-11-21T21:14:36,458 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/fba1958d70f04821b2838b549030ae7c 2024-11-21T21:14:36,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/3dd0c958ba824f80a2d1b2ceb52e7dad is 50, key is test_row_0/B:col10/1732223675838/Put/seqid=0 2024-11-21T21:14:36,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741943_1119 (size=12301) 2024-11-21T21:14:36,480 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/3dd0c958ba824f80a2d1b2ceb52e7dad 2024-11-21T21:14:36,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:36,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223736482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:36,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/84c7f8f87e5746998a095e9229e10958 is 50, key is test_row_0/C:col10/1732223675838/Put/seqid=0 2024-11-21T21:14:36,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:36,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:36,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:36,491 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732223676464, max timestamp=1732223676487Num uniques 859; 2024-11-21T21:14:36,491 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-21T21:14:36,493 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732223676464, max timestamp=1732223676487Num uniques 859; 2024-11-21T21:14:36,493 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-21T21:14:36,495 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732223676464, max timestamp=1732223676487Num uniques 859; 2024-11-21T21:14:36,495 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-21T21:14:36,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741944_1120 (size=12301) 2024-11-21T21:14:36,507 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/84c7f8f87e5746998a095e9229e10958 2024-11-21T21:14:36,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/fba1958d70f04821b2838b549030ae7c as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/fba1958d70f04821b2838b549030ae7c 2024-11-21T21:14:36,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:36,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223736514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:36,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:36,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:36,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-21T21:14:36,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:36,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:36,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:36,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:36,520 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:36,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:36,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:36,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:36,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223736518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:36,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:36,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:36,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:36,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:36,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223736518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:36,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:36,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223736519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:36,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:36,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:36,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:36,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:36,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:36,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:36,528 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/fba1958d70f04821b2838b549030ae7c, entries=250, sequenceid=571, filesize=16.8 K 2024-11-21T21:14:36,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/3dd0c958ba824f80a2d1b2ceb52e7dad as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/3dd0c958ba824f80a2d1b2ceb52e7dad 2024-11-21T21:14:36,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T21:14:36,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/3dd0c958ba824f80a2d1b2ceb52e7dad, entries=150, sequenceid=571, filesize=12.0 K 2024-11-21T21:14:36,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/84c7f8f87e5746998a095e9229e10958 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/84c7f8f87e5746998a095e9229e10958 2024-11-21T21:14:36,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/84c7f8f87e5746998a095e9229e10958, entries=150, sequenceid=571, filesize=12.0 K 2024-11-21T21:14:36,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=140.89 KB/144270 for ae951cc3e18af3209220d547b1b63947 in 100ms, sequenceid=571, compaction requested=true 2024-11-21T21:14:36,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:36,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae951cc3e18af3209220d547b1b63947:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:14:36,543 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:14:36,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:36,543 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:14:36,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae951cc3e18af3209220d547b1b63947:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:14:36,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:36,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae951cc3e18af3209220d547b1b63947:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:14:36,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:14:36,544 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:14:36,544 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): ae951cc3e18af3209220d547b1b63947/B is initiating minor compaction (all files) 2024-11-21T21:14:36,545 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae951cc3e18af3209220d547b1b63947/B in TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:36,545 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/7fe4b4f991294d0d9901abeb41c025af, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/fdd24163bb0e46f3ba8b607ec635d897, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/3dd0c958ba824f80a2d1b2ceb52e7dad] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp, totalSize=37.1 K 2024-11-21T21:14:36,545 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 50227 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:14:36,545 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): ae951cc3e18af3209220d547b1b63947/A is initiating minor compaction (all files) 2024-11-21T21:14:36,545 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae951cc3e18af3209220d547b1b63947/A in TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:36,545 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/607b9e2d69434a62a54085993ef14ed0, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/070c4c3807fc4c4e8fe62a583171c90a, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/fba1958d70f04821b2838b549030ae7c] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp, totalSize=49.0 K 2024-11-21T21:14:36,546 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fe4b4f991294d0d9901abeb41c025af, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1732223675414 2024-11-21T21:14:36,546 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 607b9e2d69434a62a54085993ef14ed0, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1732223675414 2024-11-21T21:14:36,546 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting fdd24163bb0e46f3ba8b607ec635d897, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=557, earliestPutTs=1732223675823 2024-11-21T21:14:36,546 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 070c4c3807fc4c4e8fe62a583171c90a, keycount=300, bloomtype=ROW, size=19.2 K, encoding=NONE, compression=NONE, seqNum=557, earliestPutTs=1732223675823 2024-11-21T21:14:36,547 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting fba1958d70f04821b2838b549030ae7c, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=571, earliestPutTs=1732223675838 2024-11-21T21:14:36,547 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 3dd0c958ba824f80a2d1b2ceb52e7dad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=571, earliestPutTs=1732223675838 2024-11-21T21:14:36,557 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae951cc3e18af3209220d547b1b63947#B#compaction#108 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:14:36,558 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/eaeca6aae6de4a07876da4232c9023de is 50, key is test_row_0/B:col10/1732223675838/Put/seqid=0 2024-11-21T21:14:36,559 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae951cc3e18af3209220d547b1b63947#A#compaction#109 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:14:36,560 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/ac56fa3339af4f83b4319987d2175e00 is 50, key is test_row_0/A:col10/1732223675838/Put/seqid=0 2024-11-21T21:14:36,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741945_1121 (size=13527) 2024-11-21T21:14:36,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741946_1122 (size=13527) 2024-11-21T21:14:36,571 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/ac56fa3339af4f83b4319987d2175e00 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/ac56fa3339af4f83b4319987d2175e00 2024-11-21T21:14:36,574 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/eaeca6aae6de4a07876da4232c9023de as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/eaeca6aae6de4a07876da4232c9023de 2024-11-21T21:14:36,581 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae951cc3e18af3209220d547b1b63947/A of ae951cc3e18af3209220d547b1b63947 into ac56fa3339af4f83b4319987d2175e00(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:14:36,581 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:36,581 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., storeName=ae951cc3e18af3209220d547b1b63947/A, priority=13, startTime=1732223676543; duration=0sec 2024-11-21T21:14:36,581 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:14:36,581 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae951cc3e18af3209220d547b1b63947:A 2024-11-21T21:14:36,581 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:14:36,583 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:14:36,583 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): ae951cc3e18af3209220d547b1b63947/C is initiating minor compaction (all files) 2024-11-21T21:14:36,583 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae951cc3e18af3209220d547b1b63947/C in TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:36,583 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/0298a229fbfb48bcb9bcbe6d30fc22e4, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/a2c15d8e080b42b1a32802ba3d0e176e, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/84c7f8f87e5746998a095e9229e10958] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp, totalSize=37.1 K 2024-11-21T21:14:36,585 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0298a229fbfb48bcb9bcbe6d30fc22e4, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1732223675414 2024-11-21T21:14:36,586 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2c15d8e080b42b1a32802ba3d0e176e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=557, earliestPutTs=1732223675823 2024-11-21T21:14:36,586 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84c7f8f87e5746998a095e9229e10958, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=571, earliestPutTs=1732223675838 2024-11-21T21:14:36,589 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae951cc3e18af3209220d547b1b63947/B of ae951cc3e18af3209220d547b1b63947 into eaeca6aae6de4a07876da4232c9023de(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:14:36,589 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:36,589 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., storeName=ae951cc3e18af3209220d547b1b63947/B, priority=13, startTime=1732223676543; duration=0sec 2024-11-21T21:14:36,589 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:36,590 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae951cc3e18af3209220d547b1b63947:B 2024-11-21T21:14:36,596 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae951cc3e18af3209220d547b1b63947#C#compaction#110 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:14:36,597 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/7e308158069b4896a1b66d6a557176e7 is 50, key is test_row_0/C:col10/1732223675838/Put/seqid=0 2024-11-21T21:14:36,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on ae951cc3e18af3209220d547b1b63947 2024-11-21T21:14:36,603 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae951cc3e18af3209220d547b1b63947 3/3 column families, dataSize=147.60 KB heapSize=406.08 KB 2024-11-21T21:14:36,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=A 2024-11-21T21:14:36,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:14:36,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=B 2024-11-21T21:14:36,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:14:36,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=C 2024-11-21T21:14:36,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:14:36,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741947_1123 (size=13527) 2024-11-21T21:14:36,620 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/7d8f73648a7e419d8049f542ade00986 is 50, key is test_row_0/A:col10/1732223676510/Put/seqid=0 2024-11-21T21:14:36,626 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/7e308158069b4896a1b66d6a557176e7 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/7e308158069b4896a1b66d6a557176e7 2024-11-21T21:14:36,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:36,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223736630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:36,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:36,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:36,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223736631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:36,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:36,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:36,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:36,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:36,634 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:36,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:36,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223736631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:36,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:36,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:36,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:36,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223736631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:36,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:36,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:36,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223736634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:36,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:36,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:36,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:36,639 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae951cc3e18af3209220d547b1b63947/C of ae951cc3e18af3209220d547b1b63947 into 7e308158069b4896a1b66d6a557176e7(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:14:36,639 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:36,639 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., storeName=ae951cc3e18af3209220d547b1b63947/C, priority=13, startTime=1732223676543; duration=0sec 2024-11-21T21:14:36,640 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:36,640 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae951cc3e18af3209220d547b1b63947:C 2024-11-21T21:14:36,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:36,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:36,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:36,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741948_1124 (size=14741) 2024-11-21T21:14:36,646 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=608 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/7d8f73648a7e419d8049f542ade00986 2024-11-21T21:14:36,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/3f858a3be89049f185440b9bdcfc2892 is 50, key is test_row_0/B:col10/1732223676510/Put/seqid=0 2024-11-21T21:14:36,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741949_1125 (size=12301) 2024-11-21T21:14:36,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-21T21:14:36,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:36,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:36,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:36,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:36,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:36,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:36,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:36,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223736740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:36,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:36,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:36,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:36,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-21T21:14:36,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:36,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:36,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:36,830 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:36,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:36,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:36,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:36,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223736841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:36,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:36,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:36,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:36,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223736841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:36,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:36,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:36,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:36,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:36,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:36,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223736841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:36,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:36,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:36,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:36,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:36,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223736850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:36,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:36,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:36,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:36,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:36,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223736950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:36,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:36,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:36,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:36,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-21T21:14:36,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:36,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:36,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:36,983 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:36,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:36,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:37,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T21:14:37,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=608 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/3f858a3be89049f185440b9bdcfc2892 2024-11-21T21:14:37,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/ae2c3cd1292540e79c8e2b4433b62b60 is 50, key is test_row_0/C:col10/1732223676510/Put/seqid=0 2024-11-21T21:14:37,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741950_1126 (size=12301) 2024-11-21T21:14:37,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-21T21:14:37,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:37,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:37,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:37,136 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:37,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:37,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:37,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:37,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223737150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:37,153 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:37,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:37,153 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:37,153 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:37,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223737150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:37,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:37,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:37,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:37,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:37,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223737150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:37,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:37,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:37,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:37,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:37,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223737160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:37,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:37,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:37,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:37,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:37,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223737260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:37,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:37,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:37,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:37,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-21T21:14:37,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:37,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:37,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:37,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:37,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:37,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:37,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-21T21:14:37,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:37,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:37,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:37,442 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:37,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:37,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:37,485 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=608 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/ae2c3cd1292540e79c8e2b4433b62b60 2024-11-21T21:14:37,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/7d8f73648a7e419d8049f542ade00986 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/7d8f73648a7e419d8049f542ade00986 2024-11-21T21:14:37,496 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/7d8f73648a7e419d8049f542ade00986, entries=200, sequenceid=608, filesize=14.4 K 2024-11-21T21:14:37,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/3f858a3be89049f185440b9bdcfc2892 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/3f858a3be89049f185440b9bdcfc2892 2024-11-21T21:14:37,503 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/3f858a3be89049f185440b9bdcfc2892, entries=150, sequenceid=608, filesize=12.0 K 2024-11-21T21:14:37,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/ae2c3cd1292540e79c8e2b4433b62b60 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/ae2c3cd1292540e79c8e2b4433b62b60 2024-11-21T21:14:37,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/ae2c3cd1292540e79c8e2b4433b62b60, entries=150, sequenceid=608, filesize=12.0 K 2024-11-21T21:14:37,512 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~406.03 KB/415776, currentSize=40.25 KB/41220 for ae951cc3e18af3209220d547b1b63947 in 909ms, sequenceid=608, compaction requested=false 2024-11-21T21:14:37,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:37,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-21T21:14:37,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:37,595 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing ae951cc3e18af3209220d547b1b63947 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-21T21:14:37,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=A 2024-11-21T21:14:37,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:37,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=B 2024-11-21T21:14:37,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:37,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=C 2024-11-21T21:14:37,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:37,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/f1e2b15e69a54955976f94f5dd0d62cb is 50, key is test_row_0/A:col10/1732223676621/Put/seqid=0 2024-11-21T21:14:37,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741951_1127 (size=12301) 2024-11-21T21:14:37,605 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=618 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/f1e2b15e69a54955976f94f5dd0d62cb 2024-11-21T21:14:37,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/4545afac1344482ebd0794f80a6b6a82 is 50, key is test_row_0/B:col10/1732223676621/Put/seqid=0 2024-11-21T21:14:37,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741952_1128 (size=12301) 2024-11-21T21:14:37,621 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=618 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/4545afac1344482ebd0794f80a6b6a82 2024-11-21T21:14:37,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/b5b38e73ea3848f0881efe131593826a is 50, key is test_row_0/C:col10/1732223676621/Put/seqid=0 2024-11-21T21:14:37,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741953_1129 (size=12301) 2024-11-21T21:14:37,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on ae951cc3e18af3209220d547b1b63947 2024-11-21T21:14:37,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:37,688 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732223677666, max timestamp=1732223677685Num uniques 859; 2024-11-21T21:14:37,688 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732223677666, max timestamp=1732223677685Num uniques 859; 2024-11-21T21:14:37,688 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732223677666, max timestamp=1732223677685Num uniques 859; 2024-11-21T21:14:37,688 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-21T21:14:37,688 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-21T21:14:37,688 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-21T21:14:37,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:37,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223737704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:37,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:37,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:37,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:37,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:37,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223737709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:37,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:37,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:37,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:37,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223737710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:37,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:37,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:37,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:37,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:37,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:37,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223737711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:37,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:37,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:37,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:37,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:37,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223737770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:37,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:37,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:37,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:37,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:37,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:37,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223737820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:37,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223737820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:37,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:37,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:37,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:37,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:37,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:37,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223737820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:37,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:37,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:37,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:37,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:37,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:37,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:37,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223737820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:38,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:38,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223738030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:38,034 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=618 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/b5b38e73ea3848f0881efe131593826a 2024-11-21T21:14:38,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:38,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:38,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:38,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:38,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223738030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:38,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:38,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223738030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:38,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:38,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:38,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223738030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:38,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:38,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:38,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:38,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:38,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:38,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:38,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:38,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:38,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/f1e2b15e69a54955976f94f5dd0d62cb as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/f1e2b15e69a54955976f94f5dd0d62cb 2024-11-21T21:14:38,047 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/f1e2b15e69a54955976f94f5dd0d62cb, entries=150, sequenceid=618, filesize=12.0 K 2024-11-21T21:14:38,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/4545afac1344482ebd0794f80a6b6a82 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/4545afac1344482ebd0794f80a6b6a82 2024-11-21T21:14:38,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T21:14:38,056 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/4545afac1344482ebd0794f80a6b6a82, entries=150, sequenceid=618, filesize=12.0 K 2024-11-21T21:14:38,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/b5b38e73ea3848f0881efe131593826a as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/b5b38e73ea3848f0881efe131593826a 2024-11-21T21:14:38,065 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/b5b38e73ea3848f0881efe131593826a, entries=150, sequenceid=618, filesize=12.0 K 2024-11-21T21:14:38,066 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=147.60 KB/151140 for ae951cc3e18af3209220d547b1b63947 in 471ms, sequenceid=618, compaction requested=true 2024-11-21T21:14:38,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:38,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:38,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-21T21:14:38,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-21T21:14:38,070 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-21T21:14:38,070 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1620 sec 2024-11-21T21:14:38,073 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 2.1690 sec 2024-11-21T21:14:38,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on ae951cc3e18af3209220d547b1b63947 2024-11-21T21:14:38,344 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae951cc3e18af3209220d547b1b63947 3/3 column families, dataSize=154.31 KB heapSize=430.69 KB 2024-11-21T21:14:38,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=A 2024-11-21T21:14:38,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-21T21:14:38,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=B 2024-11-21T21:14:38,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-21T21:14:38,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=C 2024-11-21T21:14:38,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-21T21:14:38,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:38,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223738349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:38,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:38,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:38,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:38,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:38,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223738350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:38,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:38,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:38,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:38,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:38,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223738352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:38,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:38,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:38,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:38,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:38,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223738353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:38,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:38,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:38,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:38,396 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/fae2477060494d378a1fb9da4b790d4a is 50, key is test_row_0/A:col10/1732223678343/Put/seqid=0 2024-11-21T21:14:38,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741954_1130 (size=17181) 2024-11-21T21:14:38,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:38,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223738460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:38,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:38,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:38,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:38,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:38,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223738461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:38,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:38,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223738461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:38,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:38,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:38,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:38,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223738461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:38,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:38,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:38,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:38,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:38,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:38,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:38,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:38,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:38,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223738670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:38,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:38,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:38,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223738670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:38,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:38,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:38,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:38,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:38,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:38,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:38,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223738670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:38,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:38,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:38,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:38,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223738670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:38,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:38,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:38,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:38,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:38,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:38,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39534 deadline: 1732223738780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:38,786 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,41213,1732223653469 for 'test_row_0' in ae951cc3e18af3209220d547b1b63947 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:14:38,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:38,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:38,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:38,801 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=656 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/fae2477060494d378a1fb9da4b790d4a 2024-11-21T21:14:38,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/ea54c96773e34025a89ae1e4b03bc511 is 50, key is test_row_0/B:col10/1732223678343/Put/seqid=0 2024-11-21T21:14:38,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741955_1131 (size=12301) 2024-11-21T21:14:38,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:38,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223738980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:38,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:38,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:38,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:38,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223738980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:38,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:38,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:38,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:38,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:38,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223738980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:38,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:38,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:38,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:38,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:38,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:38,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223738980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:38,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:38,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:38,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:39,216 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=656 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/ea54c96773e34025a89ae1e4b03bc511 2024-11-21T21:14:39,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/2c02c56aaad64a0f8d78459c0567d0f1 is 50, key is test_row_0/C:col10/1732223678343/Put/seqid=0 2024-11-21T21:14:39,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741956_1132 (size=12301) 2024-11-21T21:14:39,265 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=656 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/2c02c56aaad64a0f8d78459c0567d0f1 2024-11-21T21:14:39,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/fae2477060494d378a1fb9da4b790d4a as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/fae2477060494d378a1fb9da4b790d4a 2024-11-21T21:14:39,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/fae2477060494d378a1fb9da4b790d4a, entries=250, sequenceid=656, filesize=16.8 K 2024-11-21T21:14:39,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/ea54c96773e34025a89ae1e4b03bc511 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/ea54c96773e34025a89ae1e4b03bc511 2024-11-21T21:14:39,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/ea54c96773e34025a89ae1e4b03bc511, entries=150, sequenceid=656, filesize=12.0 K 2024-11-21T21:14:39,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/2c02c56aaad64a0f8d78459c0567d0f1 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/2c02c56aaad64a0f8d78459c0567d0f1 2024-11-21T21:14:39,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/2c02c56aaad64a0f8d78459c0567d0f1, entries=150, sequenceid=656, filesize=12.0 K 2024-11-21T21:14:39,293 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~451.27 KB/462096, currentSize=26.84 KB/27480 for ae951cc3e18af3209220d547b1b63947 in 949ms, sequenceid=656, compaction requested=true 2024-11-21T21:14:39,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:39,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae951cc3e18af3209220d547b1b63947:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:14:39,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:39,294 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:14:39,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae951cc3e18af3209220d547b1b63947:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:14:39,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:39,294 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:14:39,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae951cc3e18af3209220d547b1b63947:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:14:39,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:14:39,295 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57750 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:14:39,295 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): ae951cc3e18af3209220d547b1b63947/A is initiating minor compaction (all files) 2024-11-21T21:14:39,295 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae951cc3e18af3209220d547b1b63947/A in TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:39,295 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/ac56fa3339af4f83b4319987d2175e00, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/7d8f73648a7e419d8049f542ade00986, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/f1e2b15e69a54955976f94f5dd0d62cb, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/fae2477060494d378a1fb9da4b790d4a] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp, totalSize=56.4 K 2024-11-21T21:14:39,296 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac56fa3339af4f83b4319987d2175e00, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=571, earliestPutTs=1732223675838 2024-11-21T21:14:39,296 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d8f73648a7e419d8049f542ade00986, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=608, earliestPutTs=1732223676510 2024-11-21T21:14:39,296 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50430 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:14:39,297 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): ae951cc3e18af3209220d547b1b63947/B is initiating minor compaction (all files) 2024-11-21T21:14:39,297 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1e2b15e69a54955976f94f5dd0d62cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=618, earliestPutTs=1732223676621 2024-11-21T21:14:39,297 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae951cc3e18af3209220d547b1b63947/B in TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:39,297 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/eaeca6aae6de4a07876da4232c9023de, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/3f858a3be89049f185440b9bdcfc2892, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/4545afac1344482ebd0794f80a6b6a82, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/ea54c96773e34025a89ae1e4b03bc511] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp, totalSize=49.2 K 2024-11-21T21:14:39,297 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting fae2477060494d378a1fb9da4b790d4a, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=656, earliestPutTs=1732223677703 2024-11-21T21:14:39,297 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting eaeca6aae6de4a07876da4232c9023de, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=571, earliestPutTs=1732223675838 2024-11-21T21:14:39,298 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f858a3be89049f185440b9bdcfc2892, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=608, earliestPutTs=1732223676510 2024-11-21T21:14:39,298 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 4545afac1344482ebd0794f80a6b6a82, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=618, earliestPutTs=1732223676621 2024-11-21T21:14:39,299 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting ea54c96773e34025a89ae1e4b03bc511, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=656, earliestPutTs=1732223677709 2024-11-21T21:14:39,307 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae951cc3e18af3209220d547b1b63947#A#compaction#120 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:14:39,308 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/4c720098b05646d28923f531702e0c96 is 50, key is test_row_0/A:col10/1732223678343/Put/seqid=0 2024-11-21T21:14:39,310 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae951cc3e18af3209220d547b1b63947#B#compaction#121 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:14:39,310 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/093eabffb3574fb2860511d3138e01e1 is 50, key is test_row_0/B:col10/1732223678343/Put/seqid=0 2024-11-21T21:14:39,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741957_1133 (size=13663) 2024-11-21T21:14:39,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741958_1134 (size=13663) 2024-11-21T21:14:39,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on ae951cc3e18af3209220d547b1b63947 2024-11-21T21:14:39,493 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae951cc3e18af3209220d547b1b63947 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T21:14:39,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=A 2024-11-21T21:14:39,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:39,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=B 2024-11-21T21:14:39,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:39,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=C 2024-11-21T21:14:39,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:39,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/f78ab1d30b944267a43c835561d74f69 is 50, key is test_row_0/A:col10/1732223679491/Put/seqid=0 2024-11-21T21:14:39,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741959_1135 (size=12301) 2024-11-21T21:14:39,527 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=509, cellSize=23305, totalHeapSize=53176, min timestamp=1732223679502, max timestamp=1732223679524Num uniques 859; 2024-11-21T21:14:39,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=23305, suffix heap size=117064,new segment heap size=53176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=509 2024-11-21T21:14:39,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=509, cellSize=23305, totalHeapSize=53176, min timestamp=1732223679502, max timestamp=1732223679524Num uniques 859; 2024-11-21T21:14:39,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=23305, suffix heap size=117064,new segment heap size=53176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=509 2024-11-21T21:14:39,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=509, cellSize=23305, totalHeapSize=53176, min timestamp=1732223679502, max timestamp=1732223679524Num uniques 859; 2024-11-21T21:14:39,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=23305, suffix heap size=117064,new segment heap size=53176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=509 2024-11-21T21:14:39,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:39,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223739545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:39,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:39,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:39,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223739546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:39,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:39,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:39,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:39,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:39,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:39,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223739547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:39,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:39,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:39,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:39,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:39,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:39,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223739548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:39,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:39,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:39,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:39,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:39,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223739660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:39,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:39,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:39,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:39,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:39,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:39,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223739660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:39,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223739660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:39,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:39,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:39,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:39,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:39,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:39,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223739660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:39,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:39,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:39,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:39,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:39,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:39,724 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/093eabffb3574fb2860511d3138e01e1 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/093eabffb3574fb2860511d3138e01e1 2024-11-21T21:14:39,724 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/4c720098b05646d28923f531702e0c96 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/4c720098b05646d28923f531702e0c96 2024-11-21T21:14:39,731 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae951cc3e18af3209220d547b1b63947/B of ae951cc3e18af3209220d547b1b63947 into 093eabffb3574fb2860511d3138e01e1(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:14:39,731 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae951cc3e18af3209220d547b1b63947/A of ae951cc3e18af3209220d547b1b63947 into 4c720098b05646d28923f531702e0c96(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:14:39,731 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:39,731 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:39,731 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., storeName=ae951cc3e18af3209220d547b1b63947/A, priority=12, startTime=1732223679293; duration=0sec 2024-11-21T21:14:39,731 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., storeName=ae951cc3e18af3209220d547b1b63947/B, priority=12, startTime=1732223679294; duration=0sec 2024-11-21T21:14:39,731 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:14:39,731 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae951cc3e18af3209220d547b1b63947:A 2024-11-21T21:14:39,731 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:39,731 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae951cc3e18af3209220d547b1b63947:B 2024-11-21T21:14:39,731 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:14:39,732 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50430 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:14:39,733 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): ae951cc3e18af3209220d547b1b63947/C is initiating minor compaction (all files) 2024-11-21T21:14:39,733 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae951cc3e18af3209220d547b1b63947/C in TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:39,733 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/7e308158069b4896a1b66d6a557176e7, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/ae2c3cd1292540e79c8e2b4433b62b60, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/b5b38e73ea3848f0881efe131593826a, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/2c02c56aaad64a0f8d78459c0567d0f1] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp, totalSize=49.2 K 2024-11-21T21:14:39,733 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e308158069b4896a1b66d6a557176e7, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=571, earliestPutTs=1732223675838 2024-11-21T21:14:39,734 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae2c3cd1292540e79c8e2b4433b62b60, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=608, earliestPutTs=1732223676510 2024-11-21T21:14:39,734 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5b38e73ea3848f0881efe131593826a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=618, earliestPutTs=1732223676621 2024-11-21T21:14:39,734 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c02c56aaad64a0f8d78459c0567d0f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=656, earliestPutTs=1732223677709 2024-11-21T21:14:39,748 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae951cc3e18af3209220d547b1b63947#C#compaction#123 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:14:39,748 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/bc2af83d8925479b89194381d613b8d4 is 50, key is test_row_0/C:col10/1732223678343/Put/seqid=0 2024-11-21T21:14:39,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741960_1136 (size=13663) 2024-11-21T21:14:39,758 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/bc2af83d8925479b89194381d613b8d4 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/bc2af83d8925479b89194381d613b8d4 2024-11-21T21:14:39,764 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae951cc3e18af3209220d547b1b63947/C of ae951cc3e18af3209220d547b1b63947 into bc2af83d8925479b89194381d613b8d4(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:14:39,764 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:39,764 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., storeName=ae951cc3e18af3209220d547b1b63947/C, priority=12, startTime=1732223679294; duration=0sec 2024-11-21T21:14:39,764 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:39,764 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae951cc3e18af3209220d547b1b63947:C 2024-11-21T21:14:39,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:39,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223739870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:39,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:39,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:39,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223739871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:39,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:39,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:39,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:39,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:39,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:39,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:39,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223739871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:39,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:39,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:39,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:39,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:39,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223739871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:39,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:39,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:39,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:39,907 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=666 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/f78ab1d30b944267a43c835561d74f69 2024-11-21T21:14:39,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/2b2d199932da4e748e77b1743f96994a is 50, key is test_row_0/B:col10/1732223679491/Put/seqid=0 2024-11-21T21:14:39,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741961_1137 (size=12301) 2024-11-21T21:14:40,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T21:14:40,061 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:14:40,061 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:14:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-21T21:14:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-21T21:14:40,063 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:14:40,064 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:14:40,064 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:14:40,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-21T21:14:40,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:40,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223740180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:40,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:40,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:40,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:40,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:40,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223740180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:40,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:40,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:40,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:40,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:40,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223740181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:40,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:40,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:40,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:40,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223740181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:40,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:40,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:40,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:40,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:40,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-21T21:14:40,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:40,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:40,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:40,216 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:40,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:40,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:40,322 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=666 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/2b2d199932da4e748e77b1743f96994a 2024-11-21T21:14:40,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/4abdac32d24c4ba49449e3c6ff1abde0 is 50, key is test_row_0/C:col10/1732223679491/Put/seqid=0 2024-11-21T21:14:40,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741962_1138 (size=12301) 2024-11-21T21:14:40,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-21T21:14:40,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:40,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:40,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:40,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:40,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:40,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:40,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-21T21:14:40,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-21T21:14:40,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:40,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:40,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:40,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:40,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:40,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:40,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-21T21:14:40,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:40,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:40,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:40,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:40,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:40,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:40,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-21T21:14:40,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:40,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39512 deadline: 1732223740690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:40,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:40,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:40,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:40,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:40,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39510 deadline: 1732223740690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:40,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:40,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39532 deadline: 1732223740690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:40,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:40,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:40,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:40,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:40,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:40,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:40,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39536 deadline: 1732223740690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:40,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:40,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:40,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:40,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:14:40,705 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-21T21:14:40,705 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:14:40,707 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-21T21:14:40,707 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:14:40,709 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:14:40,709 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:14:40,710 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-21T21:14:40,710 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:14:40,710 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:14:40,710 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:14:40,710 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:14:40,710 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:14:40,710 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:14:40,710 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:14:40,710 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:14:40,711 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-21T21:14:40,711 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:14:40,711 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:14:40,711 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:14:40,711 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:14:40,735 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=666 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/4abdac32d24c4ba49449e3c6ff1abde0 2024-11-21T21:14:40,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/f78ab1d30b944267a43c835561d74f69 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/f78ab1d30b944267a43c835561d74f69 2024-11-21T21:14:40,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/f78ab1d30b944267a43c835561d74f69, entries=150, sequenceid=666, filesize=12.0 K 2024-11-21T21:14:40,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/2b2d199932da4e748e77b1743f96994a as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/2b2d199932da4e748e77b1743f96994a 2024-11-21T21:14:40,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/2b2d199932da4e748e77b1743f96994a, entries=150, sequenceid=666, filesize=12.0 K 2024-11-21T21:14:40,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/4abdac32d24c4ba49449e3c6ff1abde0 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/4abdac32d24c4ba49449e3c6ff1abde0 2024-11-21T21:14:40,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/4abdac32d24c4ba49449e3c6ff1abde0, entries=150, sequenceid=666, filesize=12.0 K 2024-11-21T21:14:40,763 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for ae951cc3e18af3209220d547b1b63947 in 1270ms, sequenceid=666, compaction requested=false 2024-11-21T21:14:40,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:40,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on ae951cc3e18af3209220d547b1b63947 2024-11-21T21:14:40,793 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae951cc3e18af3209220d547b1b63947 3/3 column families, dataSize=147.60 KB heapSize=392.02 KB 2024-11-21T21:14:40,793 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-21T21:14:40,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=A 2024-11-21T21:14:40,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:14:40,793 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:14:40,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=B 2024-11-21T21:14:40,794 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:14:40,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:14:40,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=C 2024-11-21T21:14:40,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:14:40,794 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:14:40,794 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:14:40,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/b7b3a45504014f3a87d30ef7af2b9f0a is 50, key is test_row_0/A:col10/1732223680791/Put/seqid=0 2024-11-21T21:14:40,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741963_1139 (size=12301) 2024-11-21T21:14:40,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-21T21:14:40,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:40,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:40,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:40,830 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:40,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:40,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:40,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-21T21:14:40,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:40,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:40,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:40,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:40,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:40,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:41,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-21T21:14:41,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:41,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:41,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:41,143 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:41,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:41,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:41,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-21T21:14:41,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=701 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/b7b3a45504014f3a87d30ef7af2b9f0a 2024-11-21T21:14:41,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/c9e995f2dda24b1da35e94c4e2135e4c is 50, key is test_row_0/B:col10/1732223680791/Put/seqid=0 2024-11-21T21:14:41,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741964_1140 (size=12301) 2024-11-21T21:14:41,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-21T21:14:41,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:41,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:41,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:41,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:41,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:41,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:41,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-21T21:14:41,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:41,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:41,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:41,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:41,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:41,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:41,603 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T21:14:41,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-21T21:14:41,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:41,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:41,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:41,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:41,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:41,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:41,628 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=701 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/c9e995f2dda24b1da35e94c4e2135e4c 2024-11-21T21:14:41,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/97dbed132870400ca7aa73587bb656c0 is 50, key is test_row_0/C:col10/1732223680791/Put/seqid=0 2024-11-21T21:14:41,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741965_1141 (size=12301) 2024-11-21T21:14:41,702 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-21T21:14:41,703 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:14:41,703 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:14:41,703 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:14:41,703 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:14:41,712 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-21T21:14:41,712 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-21T21:14:41,712 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-21T21:14:41,712 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:14:41,713 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:14:41,712 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:14:41,713 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:14:41,713 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:14:41,713 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:14:41,713 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:14:41,713 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:14:41,713 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:14:41,713 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:14:41,713 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:14:41,713 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:14:41,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-21T21:14:41,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:41,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:41,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:41,761 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:41,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:41,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:41,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-21T21:14:41,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:41,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:41,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:41,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:41,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:41,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:42,049 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=701 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/97dbed132870400ca7aa73587bb656c0 2024-11-21T21:14:42,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/b7b3a45504014f3a87d30ef7af2b9f0a as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/b7b3a45504014f3a87d30ef7af2b9f0a 2024-11-21T21:14:42,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/b7b3a45504014f3a87d30ef7af2b9f0a, entries=150, sequenceid=701, filesize=12.0 K 2024-11-21T21:14:42,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/c9e995f2dda24b1da35e94c4e2135e4c as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/c9e995f2dda24b1da35e94c4e2135e4c 2024-11-21T21:14:42,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/c9e995f2dda24b1da35e94c4e2135e4c, entries=150, sequenceid=701, filesize=12.0 K 2024-11-21T21:14:42,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/97dbed132870400ca7aa73587bb656c0 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/97dbed132870400ca7aa73587bb656c0 2024-11-21T21:14:42,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-21T21:14:42,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:42,074 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/97dbed132870400ca7aa73587bb656c0, entries=150, sequenceid=701, filesize=12.0 K 2024-11-21T21:14:42,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. as already flushing 2024-11-21T21:14:42,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:42,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:42,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:42,075 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~391.97 KB/401376, currentSize=26.84 KB/27480 for ae951cc3e18af3209220d547b1b63947 in 1282ms, sequenceid=701, compaction requested=true 2024-11-21T21:14:42,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:42,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae951cc3e18af3209220d547b1b63947:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:14:42,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:42,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:42,075 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:14:42,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae951cc3e18af3209220d547b1b63947:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:14:42,076 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:14:42,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:42,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae951cc3e18af3209220d547b1b63947:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:14:42,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:14:42,077 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:14:42,077 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:14:42,077 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): ae951cc3e18af3209220d547b1b63947/B is initiating minor compaction (all files) 2024-11-21T21:14:42,077 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): ae951cc3e18af3209220d547b1b63947/A is initiating minor compaction (all files) 2024-11-21T21:14:42,077 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae951cc3e18af3209220d547b1b63947/B in TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:42,077 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae951cc3e18af3209220d547b1b63947/A in TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:42,077 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/093eabffb3574fb2860511d3138e01e1, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/2b2d199932da4e748e77b1743f96994a, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/c9e995f2dda24b1da35e94c4e2135e4c] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp, totalSize=37.4 K 2024-11-21T21:14:42,077 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/4c720098b05646d28923f531702e0c96, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/f78ab1d30b944267a43c835561d74f69, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/b7b3a45504014f3a87d30ef7af2b9f0a] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp, totalSize=37.4 K 2024-11-21T21:14:42,077 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 093eabffb3574fb2860511d3138e01e1, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=656, earliestPutTs=1732223677709 2024-11-21T21:14:42,077 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c720098b05646d28923f531702e0c96, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=656, earliestPutTs=1732223677709 2024-11-21T21:14:42,078 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b2d199932da4e748e77b1743f96994a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=666, earliestPutTs=1732223678350 2024-11-21T21:14:42,078 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting f78ab1d30b944267a43c835561d74f69, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=666, earliestPutTs=1732223678350 2024-11-21T21:14:42,078 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting c9e995f2dda24b1da35e94c4e2135e4c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=701, earliestPutTs=1732223679544 2024-11-21T21:14:42,078 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7b3a45504014f3a87d30ef7af2b9f0a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=701, earliestPutTs=1732223679544 2024-11-21T21:14:42,086 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae951cc3e18af3209220d547b1b63947#A#compaction#129 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:14:42,086 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae951cc3e18af3209220d547b1b63947#B#compaction#130 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:14:42,086 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/10d66027287e4c548f86f4691b35eeb8 is 50, key is test_row_0/A:col10/1732223680791/Put/seqid=0 2024-11-21T21:14:42,086 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/fcb43cf0455f4b88937ad7ec90614840 is 50, key is test_row_0/B:col10/1732223680791/Put/seqid=0 2024-11-21T21:14:42,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741967_1143 (size=13765) 2024-11-21T21:14:42,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741966_1142 (size=13765) 2024-11-21T21:14:42,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-21T21:14:42,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-21T21:14:42,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:42,228 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing ae951cc3e18af3209220d547b1b63947 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-21T21:14:42,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=A 2024-11-21T21:14:42,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:42,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=B 2024-11-21T21:14:42,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:42,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae951cc3e18af3209220d547b1b63947, store=C 2024-11-21T21:14:42,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:42,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/2da3d0ff0b7945e78dca732c5515aaf7 is 50, key is test_row_0/A:col10/1732223681711/Put/seqid=0 2024-11-21T21:14:42,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741968_1144 (size=12301) 2024-11-21T21:14:42,501 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/fcb43cf0455f4b88937ad7ec90614840 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/fcb43cf0455f4b88937ad7ec90614840 2024-11-21T21:14:42,501 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/10d66027287e4c548f86f4691b35eeb8 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/10d66027287e4c548f86f4691b35eeb8 2024-11-21T21:14:42,508 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae951cc3e18af3209220d547b1b63947/B of ae951cc3e18af3209220d547b1b63947 into fcb43cf0455f4b88937ad7ec90614840(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:14:42,508 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae951cc3e18af3209220d547b1b63947/A of ae951cc3e18af3209220d547b1b63947 into 10d66027287e4c548f86f4691b35eeb8(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:14:42,508 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:42,508 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:42,508 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., storeName=ae951cc3e18af3209220d547b1b63947/A, priority=13, startTime=1732223682075; duration=0sec 2024-11-21T21:14:42,508 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., storeName=ae951cc3e18af3209220d547b1b63947/B, priority=13, startTime=1732223682075; duration=0sec 2024-11-21T21:14:42,508 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:14:42,508 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae951cc3e18af3209220d547b1b63947:A 2024-11-21T21:14:42,508 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:14:42,508 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae951cc3e18af3209220d547b1b63947:B 2024-11-21T21:14:42,508 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:14:42,510 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:14:42,510 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): ae951cc3e18af3209220d547b1b63947/C is initiating minor compaction (all files) 2024-11-21T21:14:42,510 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae951cc3e18af3209220d547b1b63947/C in TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:42,510 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/bc2af83d8925479b89194381d613b8d4, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/4abdac32d24c4ba49449e3c6ff1abde0, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/97dbed132870400ca7aa73587bb656c0] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp, totalSize=37.4 K 2024-11-21T21:14:42,510 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc2af83d8925479b89194381d613b8d4, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=656, earliestPutTs=1732223677709 2024-11-21T21:14:42,511 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4abdac32d24c4ba49449e3c6ff1abde0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=666, earliestPutTs=1732223678350 2024-11-21T21:14:42,511 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97dbed132870400ca7aa73587bb656c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=701, earliestPutTs=1732223679544 2024-11-21T21:14:42,519 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae951cc3e18af3209220d547b1b63947#C#compaction#132 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:14:42,520 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/a5b40cfa29d64a7d9210dc99bc5507f8 is 50, key is test_row_0/C:col10/1732223680791/Put/seqid=0 2024-11-21T21:14:42,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741969_1145 (size=13765) 2024-11-21T21:14:42,637 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=708 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/2da3d0ff0b7945e78dca732c5515aaf7 2024-11-21T21:14:42,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/855eda78710748728441553e7df3c6bb is 50, key is test_row_0/B:col10/1732223681711/Put/seqid=0 2024-11-21T21:14:42,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741970_1146 (size=12301) 2024-11-21T21:14:42,927 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/a5b40cfa29d64a7d9210dc99bc5507f8 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/a5b40cfa29d64a7d9210dc99bc5507f8 2024-11-21T21:14:42,933 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae951cc3e18af3209220d547b1b63947/C of ae951cc3e18af3209220d547b1b63947 into a5b40cfa29d64a7d9210dc99bc5507f8(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:14:42,933 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:42,933 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947., storeName=ae951cc3e18af3209220d547b1b63947/C, priority=13, startTime=1732223682076; duration=0sec 2024-11-21T21:14:42,933 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:42,933 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae951cc3e18af3209220d547b1b63947:C 2024-11-21T21:14:43,051 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=708 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/855eda78710748728441553e7df3c6bb 2024-11-21T21:14:43,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/3155eb1239b4446b81f2d3a240fb5451 is 50, key is test_row_0/C:col10/1732223681711/Put/seqid=0 2024-11-21T21:14:43,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741971_1147 (size=12301) 2024-11-21T21:14:43,467 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=708 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/3155eb1239b4446b81f2d3a240fb5451 2024-11-21T21:14:43,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/A/2da3d0ff0b7945e78dca732c5515aaf7 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/2da3d0ff0b7945e78dca732c5515aaf7 2024-11-21T21:14:43,485 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/2da3d0ff0b7945e78dca732c5515aaf7, entries=150, sequenceid=708, filesize=12.0 K 2024-11-21T21:14:43,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/B/855eda78710748728441553e7df3c6bb as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/855eda78710748728441553e7df3c6bb 2024-11-21T21:14:43,491 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/855eda78710748728441553e7df3c6bb, entries=150, sequenceid=708, filesize=12.0 K 2024-11-21T21:14:43,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/.tmp/C/3155eb1239b4446b81f2d3a240fb5451 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/3155eb1239b4446b81f2d3a240fb5451 2024-11-21T21:14:43,496 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/3155eb1239b4446b81f2d3a240fb5451, entries=150, sequenceid=708, filesize=12.0 K 2024-11-21T21:14:43,497 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for ae951cc3e18af3209220d547b1b63947 in 1269ms, sequenceid=708, compaction requested=false 2024-11-21T21:14:43,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for ae951cc3e18af3209220d547b1b63947: 2024-11-21T21:14:43,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:43,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-21T21:14:43,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-21T21:14:43,500 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-21T21:14:43,501 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4350 sec 2024-11-21T21:14:43,502 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 3.4400 sec 2024-11-21T21:14:44,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-21T21:14:44,251 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:14:44,251 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T21:14:44,252 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 84 2024-11-21T21:14:44,252 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 124 2024-11-21T21:14:44,252 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 129 2024-11-21T21:14:44,252 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 137 2024-11-21T21:14:44,252 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 101 2024-11-21T21:14:44,252 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T21:14:44,252 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2751 2024-11-21T21:14:44,252 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2642 2024-11-21T21:14:44,253 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T21:14:44,253 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1187 2024-11-21T21:14:44,253 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3560 rows 2024-11-21T21:14:44,253 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1184 2024-11-21T21:14:44,253 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3546 rows 2024-11-21T21:14:44,254 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T21:14:44,254 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T21:14:44,255 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:14:44,255 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:14:44,255 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:14:44,262 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-21T21:14:44,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T21:14:44,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-21T21:14:44,268 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732223684268"}]},"ts":"1732223684268"} 2024-11-21T21:14:44,270 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T21:14:44,270 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T21:14:44,272 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T21:14:44,276 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae951cc3e18af3209220d547b1b63947, UNASSIGN}] 2024-11-21T21:14:44,277 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae951cc3e18af3209220d547b1b63947, UNASSIGN 2024-11-21T21:14:44,278 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=ae951cc3e18af3209220d547b1b63947, regionState=CLOSING, regionLocation=849dee24a242,41213,1732223653469 2024-11-21T21:14:44,280 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae951cc3e18af3209220d547b1b63947, UNASSIGN because future has completed 2024-11-21T21:14:44,281 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T21:14:44,281 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469}] 2024-11-21T21:14:44,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-21T21:14:44,438 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close ae951cc3e18af3209220d547b1b63947 2024-11-21T21:14:44,438 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T21:14:44,439 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing ae951cc3e18af3209220d547b1b63947, disabling compactions & flushes 2024-11-21T21:14:44,439 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:44,439 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:44,439 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. after waiting 0 ms 2024-11-21T21:14:44,439 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:44,440 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/6acc1e84ea42446cafd2f800780b19b4, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/7b1d67a56ac34d93b9a119f8b3d3f8c4, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/059758ef32d74cc7aa6f0403af5717a7, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/903c4e60d57948db94a65a96331e9c7f, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/223636bcf5664d01b47f4ad6eec2f405, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/653f2f5b1c094038a95f14af19f28bb7, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/968a26425ab746069a9004f41a5719ea, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/f0e1c06ad55b40a9860bb148af5a70d0, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/1996f79146a44a26aa1f4e8b0a96b5f7, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/aa6e505ecf4c4193827417ef67a2ec19, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/d48b357108a34a5083b680e6f9130b86, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/d296505f965041929408b93f3b2415a4, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/240e1bcd45df4ae1a0e29f091ab8813a, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/62db0869e8a044738693d02913cbfa9a, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/04f6461152e941ae8b8476242acd278b, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/221e4956b19343c78dcd8b154be99f00, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/c7d27b799d784851997079ab12b90a82, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/0b18fb4120ae4f429df4263e21e98105, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/42032c7c555746a69a461e5e55f85f0a, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/e2b68f3ff39e4a08b964c5bf6f11ba42, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/2914a3db94654c3890a542f5450d6cc5, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/92a6c06c735640c0ba29fe99250d248f, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/72070a5fc8b04ffeba7a4dd35a990682, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/26aa18e0409149f3a5ba94de8671c301, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/7a8088da3899455cb2ebaeb59e4a0286, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/f44ffca992914049bd44a73afcd24768, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/e146707210234ffca6d269ce49d7f4f7, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/eeb709c8e18644adaf73f02a1bc6b096, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/09284d4fd3204cd3be4347922948d1b9, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/c4c56e395c6f4a6d862a2442ff76666e, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/6a7162851c334e4dbf2061e204d72b04, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/c06580df8f724fb39bdcd8b4052c1f1c, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/334e7d2d1d20497b89f46fefef4544a3, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/607b9e2d69434a62a54085993ef14ed0, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/070c4c3807fc4c4e8fe62a583171c90a, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/fba1958d70f04821b2838b549030ae7c, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/ac56fa3339af4f83b4319987d2175e00, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/7d8f73648a7e419d8049f542ade00986, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/f1e2b15e69a54955976f94f5dd0d62cb, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/fae2477060494d378a1fb9da4b790d4a, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/4c720098b05646d28923f531702e0c96, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/f78ab1d30b944267a43c835561d74f69, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/b7b3a45504014f3a87d30ef7af2b9f0a] to archive 2024-11-21T21:14:44,444 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T21:14:44,452 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/6acc1e84ea42446cafd2f800780b19b4 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/6acc1e84ea42446cafd2f800780b19b4 2024-11-21T21:14:44,454 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/7b1d67a56ac34d93b9a119f8b3d3f8c4 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/7b1d67a56ac34d93b9a119f8b3d3f8c4 2024-11-21T21:14:44,455 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/059758ef32d74cc7aa6f0403af5717a7 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/059758ef32d74cc7aa6f0403af5717a7 2024-11-21T21:14:44,457 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/903c4e60d57948db94a65a96331e9c7f to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/903c4e60d57948db94a65a96331e9c7f 2024-11-21T21:14:44,459 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/223636bcf5664d01b47f4ad6eec2f405 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/223636bcf5664d01b47f4ad6eec2f405 2024-11-21T21:14:44,461 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/653f2f5b1c094038a95f14af19f28bb7 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/653f2f5b1c094038a95f14af19f28bb7 2024-11-21T21:14:44,462 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/968a26425ab746069a9004f41a5719ea to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/968a26425ab746069a9004f41a5719ea 2024-11-21T21:14:44,464 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/f0e1c06ad55b40a9860bb148af5a70d0 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/f0e1c06ad55b40a9860bb148af5a70d0 2024-11-21T21:14:44,466 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/1996f79146a44a26aa1f4e8b0a96b5f7 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/1996f79146a44a26aa1f4e8b0a96b5f7 2024-11-21T21:14:44,468 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/aa6e505ecf4c4193827417ef67a2ec19 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/aa6e505ecf4c4193827417ef67a2ec19 2024-11-21T21:14:44,469 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/d48b357108a34a5083b680e6f9130b86 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/d48b357108a34a5083b680e6f9130b86 2024-11-21T21:14:44,471 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/d296505f965041929408b93f3b2415a4 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/d296505f965041929408b93f3b2415a4 2024-11-21T21:14:44,473 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/240e1bcd45df4ae1a0e29f091ab8813a to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/240e1bcd45df4ae1a0e29f091ab8813a 2024-11-21T21:14:44,475 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/62db0869e8a044738693d02913cbfa9a to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/62db0869e8a044738693d02913cbfa9a 2024-11-21T21:14:44,476 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/04f6461152e941ae8b8476242acd278b to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/04f6461152e941ae8b8476242acd278b 2024-11-21T21:14:44,478 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/221e4956b19343c78dcd8b154be99f00 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/221e4956b19343c78dcd8b154be99f00 2024-11-21T21:14:44,480 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/c7d27b799d784851997079ab12b90a82 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/c7d27b799d784851997079ab12b90a82 2024-11-21T21:14:44,482 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/0b18fb4120ae4f429df4263e21e98105 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/0b18fb4120ae4f429df4263e21e98105 2024-11-21T21:14:44,483 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/42032c7c555746a69a461e5e55f85f0a to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/42032c7c555746a69a461e5e55f85f0a 2024-11-21T21:14:44,485 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/e2b68f3ff39e4a08b964c5bf6f11ba42 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/e2b68f3ff39e4a08b964c5bf6f11ba42 2024-11-21T21:14:44,486 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/2914a3db94654c3890a542f5450d6cc5 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/2914a3db94654c3890a542f5450d6cc5 2024-11-21T21:14:44,487 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/92a6c06c735640c0ba29fe99250d248f to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/92a6c06c735640c0ba29fe99250d248f 2024-11-21T21:14:44,488 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/72070a5fc8b04ffeba7a4dd35a990682 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/72070a5fc8b04ffeba7a4dd35a990682 2024-11-21T21:14:44,489 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/26aa18e0409149f3a5ba94de8671c301 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/26aa18e0409149f3a5ba94de8671c301 2024-11-21T21:14:44,490 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/7a8088da3899455cb2ebaeb59e4a0286 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/7a8088da3899455cb2ebaeb59e4a0286 2024-11-21T21:14:44,491 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/f44ffca992914049bd44a73afcd24768 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/f44ffca992914049bd44a73afcd24768 2024-11-21T21:14:44,493 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/e146707210234ffca6d269ce49d7f4f7 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/e146707210234ffca6d269ce49d7f4f7 2024-11-21T21:14:44,494 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/eeb709c8e18644adaf73f02a1bc6b096 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/eeb709c8e18644adaf73f02a1bc6b096 2024-11-21T21:14:44,495 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/09284d4fd3204cd3be4347922948d1b9 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/09284d4fd3204cd3be4347922948d1b9 2024-11-21T21:14:44,496 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/c4c56e395c6f4a6d862a2442ff76666e to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/c4c56e395c6f4a6d862a2442ff76666e 2024-11-21T21:14:44,497 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/6a7162851c334e4dbf2061e204d72b04 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/6a7162851c334e4dbf2061e204d72b04 2024-11-21T21:14:44,499 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/c06580df8f724fb39bdcd8b4052c1f1c to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/c06580df8f724fb39bdcd8b4052c1f1c 2024-11-21T21:14:44,500 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/334e7d2d1d20497b89f46fefef4544a3 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/334e7d2d1d20497b89f46fefef4544a3 2024-11-21T21:14:44,501 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/607b9e2d69434a62a54085993ef14ed0 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/607b9e2d69434a62a54085993ef14ed0 2024-11-21T21:14:44,502 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/070c4c3807fc4c4e8fe62a583171c90a to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/070c4c3807fc4c4e8fe62a583171c90a 2024-11-21T21:14:44,503 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/fba1958d70f04821b2838b549030ae7c to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/fba1958d70f04821b2838b549030ae7c 2024-11-21T21:14:44,504 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/ac56fa3339af4f83b4319987d2175e00 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/ac56fa3339af4f83b4319987d2175e00 2024-11-21T21:14:44,505 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/7d8f73648a7e419d8049f542ade00986 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/7d8f73648a7e419d8049f542ade00986 2024-11-21T21:14:44,506 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/f1e2b15e69a54955976f94f5dd0d62cb to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/f1e2b15e69a54955976f94f5dd0d62cb 2024-11-21T21:14:44,508 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/fae2477060494d378a1fb9da4b790d4a to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/fae2477060494d378a1fb9da4b790d4a 2024-11-21T21:14:44,509 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/4c720098b05646d28923f531702e0c96 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/4c720098b05646d28923f531702e0c96 2024-11-21T21:14:44,510 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/f78ab1d30b944267a43c835561d74f69 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/f78ab1d30b944267a43c835561d74f69 2024-11-21T21:14:44,511 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/b7b3a45504014f3a87d30ef7af2b9f0a to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/b7b3a45504014f3a87d30ef7af2b9f0a 2024-11-21T21:14:44,523 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/fe2594045ed34001a4bcc82eda2e7bc8, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/d4bda57f98b24ff9b87d1e60f9053913, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/b37dcc48c61e4cc8a90322bfcbcee6a5, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/cdbe0680a3714d6b9aa700a9f9fccfbd, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/2cb5b396ffda4221961aa2a226a9a390, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/3cedb04748894fd1bd65a4f4db283444, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/9a0a43faea614ee3a1005cf7dca35ae3, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/17f81127c74048aba81497f87bd5e668, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/8808ec075f354e62895ce342b275cf0f, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/e8fea30b633a4cd18f1b3511c309116f, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/5f7dd2823e6c429da117b3fbb9a9535e, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/f854f6c9ae7349358543755116602d4a, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/7ed0b3b9fbb6466db0e2274a8f7b7db9, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/e7ad00743dbe44bea1e0b7cfe4aeea92, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/061b12adb0c74f1b802323cb6a85fc6d, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/40724a1d8d9d4bcb9c4e8fb6e8e6be4e, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/235fcd71c5cc4b299ab2c4883baf5ece, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/fddac71f19a4480f86558a1d27e46f84, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/1089d7e37d2a4c3885839c69d7f760f6, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/8d54d81eb3b84fa382954aaab774ed14, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/b5c8d3fbe6834fb49a4e8853390b5baa, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/8a73c26acbd54c84a83b16db0ed60451, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/33fd4ef03c224d8cb8184abf03792943, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/5d81925eaedf43f190b119986a6332ec, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/1a04fedf6edf4c11afdbefcffcf07846, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/fb0bfff8841b40e39ef57b9d09211b75, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/e3df436333b948bb9112ab1c9316f327, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/0bc8d70ee6b9473cbdb431b482380b2c, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/2e07dd17bdcc4ac6bfe7a5341378c1ce, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/66f2a9db07d241b4848a7055d2be8ccb, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/2821679f2896432d97f10e5cc6b1216e, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/9008963171ec4606b2785e91fab6d690, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/7fe4b4f991294d0d9901abeb41c025af, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/07034f51312b409a8e347a88d64bb381, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/fdd24163bb0e46f3ba8b607ec635d897, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/eaeca6aae6de4a07876da4232c9023de, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/3dd0c958ba824f80a2d1b2ceb52e7dad, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/3f858a3be89049f185440b9bdcfc2892, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/4545afac1344482ebd0794f80a6b6a82, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/093eabffb3574fb2860511d3138e01e1, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/ea54c96773e34025a89ae1e4b03bc511, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/2b2d199932da4e748e77b1743f96994a, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/c9e995f2dda24b1da35e94c4e2135e4c] to archive 2024-11-21T21:14:44,524 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T21:14:44,526 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/fe2594045ed34001a4bcc82eda2e7bc8 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/fe2594045ed34001a4bcc82eda2e7bc8 2024-11-21T21:14:44,528 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/d4bda57f98b24ff9b87d1e60f9053913 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/d4bda57f98b24ff9b87d1e60f9053913 2024-11-21T21:14:44,529 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/b37dcc48c61e4cc8a90322bfcbcee6a5 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/b37dcc48c61e4cc8a90322bfcbcee6a5 2024-11-21T21:14:44,530 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/cdbe0680a3714d6b9aa700a9f9fccfbd to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/cdbe0680a3714d6b9aa700a9f9fccfbd 2024-11-21T21:14:44,531 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/2cb5b396ffda4221961aa2a226a9a390 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/2cb5b396ffda4221961aa2a226a9a390 2024-11-21T21:14:44,532 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/3cedb04748894fd1bd65a4f4db283444 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/3cedb04748894fd1bd65a4f4db283444 2024-11-21T21:14:44,533 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/9a0a43faea614ee3a1005cf7dca35ae3 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/9a0a43faea614ee3a1005cf7dca35ae3 2024-11-21T21:14:44,534 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/17f81127c74048aba81497f87bd5e668 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/17f81127c74048aba81497f87bd5e668 2024-11-21T21:14:44,535 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/8808ec075f354e62895ce342b275cf0f to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/8808ec075f354e62895ce342b275cf0f 2024-11-21T21:14:44,537 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/e8fea30b633a4cd18f1b3511c309116f to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/e8fea30b633a4cd18f1b3511c309116f 2024-11-21T21:14:44,538 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/5f7dd2823e6c429da117b3fbb9a9535e to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/5f7dd2823e6c429da117b3fbb9a9535e 2024-11-21T21:14:44,539 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/f854f6c9ae7349358543755116602d4a to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/f854f6c9ae7349358543755116602d4a 2024-11-21T21:14:44,540 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/7ed0b3b9fbb6466db0e2274a8f7b7db9 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/7ed0b3b9fbb6466db0e2274a8f7b7db9 2024-11-21T21:14:44,542 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/e7ad00743dbe44bea1e0b7cfe4aeea92 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/e7ad00743dbe44bea1e0b7cfe4aeea92 2024-11-21T21:14:44,543 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/061b12adb0c74f1b802323cb6a85fc6d to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/061b12adb0c74f1b802323cb6a85fc6d 2024-11-21T21:14:44,545 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/40724a1d8d9d4bcb9c4e8fb6e8e6be4e to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/40724a1d8d9d4bcb9c4e8fb6e8e6be4e 2024-11-21T21:14:44,546 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/235fcd71c5cc4b299ab2c4883baf5ece to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/235fcd71c5cc4b299ab2c4883baf5ece 2024-11-21T21:14:44,547 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/fddac71f19a4480f86558a1d27e46f84 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/fddac71f19a4480f86558a1d27e46f84 2024-11-21T21:14:44,549 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/1089d7e37d2a4c3885839c69d7f760f6 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/1089d7e37d2a4c3885839c69d7f760f6 2024-11-21T21:14:44,551 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/8d54d81eb3b84fa382954aaab774ed14 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/8d54d81eb3b84fa382954aaab774ed14 2024-11-21T21:14:44,554 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/b5c8d3fbe6834fb49a4e8853390b5baa to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/b5c8d3fbe6834fb49a4e8853390b5baa 2024-11-21T21:14:44,555 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/8a73c26acbd54c84a83b16db0ed60451 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/8a73c26acbd54c84a83b16db0ed60451 2024-11-21T21:14:44,556 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/33fd4ef03c224d8cb8184abf03792943 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/33fd4ef03c224d8cb8184abf03792943 2024-11-21T21:14:44,558 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/5d81925eaedf43f190b119986a6332ec to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/5d81925eaedf43f190b119986a6332ec 2024-11-21T21:14:44,559 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/1a04fedf6edf4c11afdbefcffcf07846 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/1a04fedf6edf4c11afdbefcffcf07846 2024-11-21T21:14:44,560 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/fb0bfff8841b40e39ef57b9d09211b75 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/fb0bfff8841b40e39ef57b9d09211b75 2024-11-21T21:14:44,561 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/e3df436333b948bb9112ab1c9316f327 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/e3df436333b948bb9112ab1c9316f327 2024-11-21T21:14:44,562 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/0bc8d70ee6b9473cbdb431b482380b2c to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/0bc8d70ee6b9473cbdb431b482380b2c 2024-11-21T21:14:44,563 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/2e07dd17bdcc4ac6bfe7a5341378c1ce to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/2e07dd17bdcc4ac6bfe7a5341378c1ce 2024-11-21T21:14:44,565 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/66f2a9db07d241b4848a7055d2be8ccb to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/66f2a9db07d241b4848a7055d2be8ccb 2024-11-21T21:14:44,566 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/2821679f2896432d97f10e5cc6b1216e to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/2821679f2896432d97f10e5cc6b1216e 2024-11-21T21:14:44,567 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/9008963171ec4606b2785e91fab6d690 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/9008963171ec4606b2785e91fab6d690 2024-11-21T21:14:44,569 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/7fe4b4f991294d0d9901abeb41c025af to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/7fe4b4f991294d0d9901abeb41c025af 2024-11-21T21:14:44,570 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/07034f51312b409a8e347a88d64bb381 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/07034f51312b409a8e347a88d64bb381 2024-11-21T21:14:44,571 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/fdd24163bb0e46f3ba8b607ec635d897 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/fdd24163bb0e46f3ba8b607ec635d897 2024-11-21T21:14:44,573 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/eaeca6aae6de4a07876da4232c9023de to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/eaeca6aae6de4a07876da4232c9023de 2024-11-21T21:14:44,574 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/3dd0c958ba824f80a2d1b2ceb52e7dad to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/3dd0c958ba824f80a2d1b2ceb52e7dad 2024-11-21T21:14:44,575 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/3f858a3be89049f185440b9bdcfc2892 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/3f858a3be89049f185440b9bdcfc2892 2024-11-21T21:14:44,577 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/4545afac1344482ebd0794f80a6b6a82 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/4545afac1344482ebd0794f80a6b6a82 2024-11-21T21:14:44,578 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/093eabffb3574fb2860511d3138e01e1 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/093eabffb3574fb2860511d3138e01e1 2024-11-21T21:14:44,580 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/ea54c96773e34025a89ae1e4b03bc511 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/ea54c96773e34025a89ae1e4b03bc511 2024-11-21T21:14:44,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-21T21:14:44,581 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/2b2d199932da4e748e77b1743f96994a to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/2b2d199932da4e748e77b1743f96994a 2024-11-21T21:14:44,583 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/c9e995f2dda24b1da35e94c4e2135e4c to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/c9e995f2dda24b1da35e94c4e2135e4c 2024-11-21T21:14:44,585 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/d6bd91d28dfd4d15b9bfff7294f860d9, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/d13b2fa7021642cfbd52146b9a910f67, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/0aeb8989d6cb4a84aa87d3a9e5248266, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/e6f35a9c9d744f9899d4d17d2a4e6482, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/e5da33c308dc4f48b12d2575f236cd0d, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/f9090e80448d436ba2c6fd7b5683ab0e, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/fabf841369a14de0bfed325bd274ba07, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/f1051ed0626649619563f573be753400, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/9b9b3e7b67f648a0b2ec8f3cd8d0b836, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/ae136ccd0bde44c49c7ccd1c44e1496b, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/e3725e3657ca4a8a94ac66dcf82d3cd0, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/32d12867be1346d1abea1c360b772414, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/b8e77c80b7a34404b91c3a1e721aed08, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/63196fb0b0ea475ba60295f9447ebb22, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/028d3d9dc29446a2a1e90b3a9f1426ca, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/593633d8540048baa886a565bdd368ec, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/a910630e4c784ce9be456a46afa0c443, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/eae3b0e9ff0046e79dcdd787b380850b, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/f3311a3e16ee492ea9b1b29da97c6917, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/6b65140ebaf2405ca0b387ff1340569b, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/709d40e8073f4bd6a4a202a740a41b18, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/5c738bb0c8254f74b0d111462fa7c013, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/c7d797920e5f44688edee669f51c633a, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/9fc1bc30b191424295ecc366b370a34a, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/3f936b85a7694f2bbee8a638c05fb4c9, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/015ab247541945c4804d2890f2e1848c, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/f1a3656039c1464d8d5e8bf7afd0fbca, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/7e6b40ff80374d858231eb23a895d193, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/dcf511c264544832b46491e510e9195b, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/aa3b58c115594c9ca62484e6a40b17c0, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/c09b9afebfc84071a2112c56ef66c823, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/aea7522982d94d51a0623c7d6029632a, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/0298a229fbfb48bcb9bcbe6d30fc22e4, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/df5ee71ee1eb48ff948d0d5e0d8baba1, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/a2c15d8e080b42b1a32802ba3d0e176e, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/7e308158069b4896a1b66d6a557176e7, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/84c7f8f87e5746998a095e9229e10958, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/ae2c3cd1292540e79c8e2b4433b62b60, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/b5b38e73ea3848f0881efe131593826a, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/bc2af83d8925479b89194381d613b8d4, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/2c02c56aaad64a0f8d78459c0567d0f1, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/4abdac32d24c4ba49449e3c6ff1abde0, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/97dbed132870400ca7aa73587bb656c0] to archive 2024-11-21T21:14:44,586 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T21:14:44,589 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/d6bd91d28dfd4d15b9bfff7294f860d9 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/d6bd91d28dfd4d15b9bfff7294f860d9 2024-11-21T21:14:44,590 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/d13b2fa7021642cfbd52146b9a910f67 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/d13b2fa7021642cfbd52146b9a910f67 2024-11-21T21:14:44,592 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/0aeb8989d6cb4a84aa87d3a9e5248266 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/0aeb8989d6cb4a84aa87d3a9e5248266 2024-11-21T21:14:44,593 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/e6f35a9c9d744f9899d4d17d2a4e6482 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/e6f35a9c9d744f9899d4d17d2a4e6482 2024-11-21T21:14:44,594 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/e5da33c308dc4f48b12d2575f236cd0d to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/e5da33c308dc4f48b12d2575f236cd0d 2024-11-21T21:14:44,596 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/f9090e80448d436ba2c6fd7b5683ab0e to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/f9090e80448d436ba2c6fd7b5683ab0e 2024-11-21T21:14:44,597 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/fabf841369a14de0bfed325bd274ba07 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/fabf841369a14de0bfed325bd274ba07 2024-11-21T21:14:44,598 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/f1051ed0626649619563f573be753400 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/f1051ed0626649619563f573be753400 2024-11-21T21:14:44,599 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/9b9b3e7b67f648a0b2ec8f3cd8d0b836 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/9b9b3e7b67f648a0b2ec8f3cd8d0b836 2024-11-21T21:14:44,600 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/ae136ccd0bde44c49c7ccd1c44e1496b to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/ae136ccd0bde44c49c7ccd1c44e1496b 2024-11-21T21:14:44,601 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/e3725e3657ca4a8a94ac66dcf82d3cd0 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/e3725e3657ca4a8a94ac66dcf82d3cd0 2024-11-21T21:14:44,603 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/32d12867be1346d1abea1c360b772414 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/32d12867be1346d1abea1c360b772414 2024-11-21T21:14:44,604 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/b8e77c80b7a34404b91c3a1e721aed08 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/b8e77c80b7a34404b91c3a1e721aed08 2024-11-21T21:14:44,605 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/63196fb0b0ea475ba60295f9447ebb22 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/63196fb0b0ea475ba60295f9447ebb22 2024-11-21T21:14:44,606 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/028d3d9dc29446a2a1e90b3a9f1426ca to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/028d3d9dc29446a2a1e90b3a9f1426ca 2024-11-21T21:14:44,607 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/593633d8540048baa886a565bdd368ec to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/593633d8540048baa886a565bdd368ec 2024-11-21T21:14:44,609 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/a910630e4c784ce9be456a46afa0c443 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/a910630e4c784ce9be456a46afa0c443 2024-11-21T21:14:44,610 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/eae3b0e9ff0046e79dcdd787b380850b to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/eae3b0e9ff0046e79dcdd787b380850b 2024-11-21T21:14:44,611 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/f3311a3e16ee492ea9b1b29da97c6917 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/f3311a3e16ee492ea9b1b29da97c6917 2024-11-21T21:14:44,612 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/6b65140ebaf2405ca0b387ff1340569b to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/6b65140ebaf2405ca0b387ff1340569b 2024-11-21T21:14:44,613 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/709d40e8073f4bd6a4a202a740a41b18 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/709d40e8073f4bd6a4a202a740a41b18 2024-11-21T21:14:44,614 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/5c738bb0c8254f74b0d111462fa7c013 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/5c738bb0c8254f74b0d111462fa7c013 2024-11-21T21:14:44,615 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/c7d797920e5f44688edee669f51c633a to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/c7d797920e5f44688edee669f51c633a 2024-11-21T21:14:44,616 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/9fc1bc30b191424295ecc366b370a34a to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/9fc1bc30b191424295ecc366b370a34a 2024-11-21T21:14:44,617 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/3f936b85a7694f2bbee8a638c05fb4c9 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/3f936b85a7694f2bbee8a638c05fb4c9 2024-11-21T21:14:44,618 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/015ab247541945c4804d2890f2e1848c to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/015ab247541945c4804d2890f2e1848c 2024-11-21T21:14:44,620 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/f1a3656039c1464d8d5e8bf7afd0fbca to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/f1a3656039c1464d8d5e8bf7afd0fbca 2024-11-21T21:14:44,621 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/7e6b40ff80374d858231eb23a895d193 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/7e6b40ff80374d858231eb23a895d193 2024-11-21T21:14:44,622 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/dcf511c264544832b46491e510e9195b to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/dcf511c264544832b46491e510e9195b 2024-11-21T21:14:44,624 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/aa3b58c115594c9ca62484e6a40b17c0 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/aa3b58c115594c9ca62484e6a40b17c0 2024-11-21T21:14:44,625 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/c09b9afebfc84071a2112c56ef66c823 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/c09b9afebfc84071a2112c56ef66c823 2024-11-21T21:14:44,626 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/aea7522982d94d51a0623c7d6029632a to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/aea7522982d94d51a0623c7d6029632a 2024-11-21T21:14:44,627 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/0298a229fbfb48bcb9bcbe6d30fc22e4 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/0298a229fbfb48bcb9bcbe6d30fc22e4 2024-11-21T21:14:44,629 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/df5ee71ee1eb48ff948d0d5e0d8baba1 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/df5ee71ee1eb48ff948d0d5e0d8baba1 2024-11-21T21:14:44,630 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/a2c15d8e080b42b1a32802ba3d0e176e to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/a2c15d8e080b42b1a32802ba3d0e176e 2024-11-21T21:14:44,631 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/7e308158069b4896a1b66d6a557176e7 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/7e308158069b4896a1b66d6a557176e7 2024-11-21T21:14:44,632 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/84c7f8f87e5746998a095e9229e10958 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/84c7f8f87e5746998a095e9229e10958 2024-11-21T21:14:44,633 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/ae2c3cd1292540e79c8e2b4433b62b60 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/ae2c3cd1292540e79c8e2b4433b62b60 2024-11-21T21:14:44,634 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/b5b38e73ea3848f0881efe131593826a to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/b5b38e73ea3848f0881efe131593826a 2024-11-21T21:14:44,635 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/bc2af83d8925479b89194381d613b8d4 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/bc2af83d8925479b89194381d613b8d4 2024-11-21T21:14:44,636 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/2c02c56aaad64a0f8d78459c0567d0f1 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/2c02c56aaad64a0f8d78459c0567d0f1 2024-11-21T21:14:44,638 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/4abdac32d24c4ba49449e3c6ff1abde0 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/4abdac32d24c4ba49449e3c6ff1abde0 2024-11-21T21:14:44,640 DEBUG [StoreCloser-TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/97dbed132870400ca7aa73587bb656c0 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/97dbed132870400ca7aa73587bb656c0 2024-11-21T21:14:44,646 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/recovered.edits/714.seqid, newMaxSeqId=714, maxSeqId=1 2024-11-21T21:14:44,647 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947. 2024-11-21T21:14:44,647 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for ae951cc3e18af3209220d547b1b63947: Waiting for close lock at 1732223684439Running coprocessor pre-close hooks at 1732223684439Disabling compacts and flushes for region at 1732223684439Disabling writes for close at 1732223684439Writing region close event to WAL at 1732223684642 (+203 ms)Running coprocessor post-close hooks at 1732223684647 (+5 ms)Closed at 1732223684647 2024-11-21T21:14:44,650 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed ae951cc3e18af3209220d547b1b63947 2024-11-21T21:14:44,650 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=ae951cc3e18af3209220d547b1b63947, regionState=CLOSED 2024-11-21T21:14:44,652 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 because future has completed 2024-11-21T21:14:44,655 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-21T21:14:44,656 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure ae951cc3e18af3209220d547b1b63947, server=849dee24a242,41213,1732223653469 in 373 msec 2024-11-21T21:14:44,657 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-21T21:14:44,657 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae951cc3e18af3209220d547b1b63947, UNASSIGN in 380 msec 2024-11-21T21:14:44,660 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-21T21:14:44,660 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 386 msec 2024-11-21T21:14:44,661 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732223684661"}]},"ts":"1732223684661"} 2024-11-21T21:14:44,663 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T21:14:44,663 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T21:14:44,664 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 400 msec 2024-11-21T21:14:44,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-21T21:14:44,891 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:14:44,900 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-21T21:14:44,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:14:44,908 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:14:44,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-21T21:14:44,910 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:14:44,916 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947 2024-11-21T21:14:44,921 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A, FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B, FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C, FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/recovered.edits] 2024-11-21T21:14:44,924 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/10d66027287e4c548f86f4691b35eeb8 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/10d66027287e4c548f86f4691b35eeb8 2024-11-21T21:14:44,925 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/2da3d0ff0b7945e78dca732c5515aaf7 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/A/2da3d0ff0b7945e78dca732c5515aaf7 2024-11-21T21:14:44,927 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/855eda78710748728441553e7df3c6bb to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/855eda78710748728441553e7df3c6bb 2024-11-21T21:14:44,928 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/fcb43cf0455f4b88937ad7ec90614840 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/B/fcb43cf0455f4b88937ad7ec90614840 2024-11-21T21:14:44,931 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/3155eb1239b4446b81f2d3a240fb5451 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/3155eb1239b4446b81f2d3a240fb5451 2024-11-21T21:14:44,932 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/a5b40cfa29d64a7d9210dc99bc5507f8 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/C/a5b40cfa29d64a7d9210dc99bc5507f8 2024-11-21T21:14:44,935 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/recovered.edits/714.seqid to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947/recovered.edits/714.seqid 2024-11-21T21:14:44,935 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ae951cc3e18af3209220d547b1b63947 2024-11-21T21:14:44,936 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T21:14:44,940 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:14:44,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-21T21:14:44,949 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T21:14:44,979 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T21:14:44,981 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:14:44,981 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T21:14:44,981 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732223684981"}]},"ts":"9223372036854775807"} 2024-11-21T21:14:44,985 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T21:14:44,985 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ae951cc3e18af3209220d547b1b63947, NAME => 'TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T21:14:44,986 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T21:14:44,986 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732223684986"}]},"ts":"9223372036854775807"} 2024-11-21T21:14:44,989 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T21:14:44,990 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:14:44,991 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 88 msec 2024-11-21T21:14:45,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-21T21:14:45,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T21:14:45,022 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:14:45,036 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=250 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1238330203_22 at /127.0.0.1:40504 [Waiting for operation #419] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;849dee24a242:41213-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1238330203_22 at /127.0.0.1:42986 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/849dee24a242:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1408284518_22 at /127.0.0.1:40648 [Waiting for operation #406] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/849dee24a242:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1408284518_22 at /127.0.0.1:42990 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=370 (was 304) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=11634 (was 12180) 2024-11-21T21:14:45,048 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=250, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=370, ProcessCount=11, AvailableMemoryMB=11633 2024-11-21T21:14:45,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T21:14:45,050 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T21:14:45,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T21:14:45,053 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T21:14:45,053 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:45,053 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-11-21T21:14:45,055 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T21:14:45,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-21T21:14:45,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741972_1148 (size=1061) 2024-11-21T21:14:45,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-21T21:14:45,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-21T21:14:45,465 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f 2024-11-21T21:14:45,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741973_1149 (size=53) 2024-11-21T21:14:45,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-21T21:14:45,872 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:14:45,872 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 0357b2a5ad6dd1cf7bba0f542d57baae, disabling compactions & flushes 2024-11-21T21:14:45,872 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:45,872 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:45,872 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. after waiting 0 ms 2024-11-21T21:14:45,872 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:45,872 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:45,872 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0357b2a5ad6dd1cf7bba0f542d57baae: Waiting for close lock at 1732223685872Disabling compacts and flushes for region at 1732223685872Disabling writes for close at 1732223685872Writing region close event to WAL at 1732223685872Closed at 1732223685872 2024-11-21T21:14:45,874 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T21:14:45,874 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732223685874"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732223685874"}]},"ts":"1732223685874"} 2024-11-21T21:14:45,876 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T21:14:45,877 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T21:14:45,878 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732223685877"}]},"ts":"1732223685877"} 2024-11-21T21:14:45,880 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T21:14:45,881 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0357b2a5ad6dd1cf7bba0f542d57baae, ASSIGN}] 2024-11-21T21:14:45,882 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0357b2a5ad6dd1cf7bba0f542d57baae, ASSIGN 2024-11-21T21:14:45,883 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0357b2a5ad6dd1cf7bba0f542d57baae, ASSIGN; state=OFFLINE, location=849dee24a242,41213,1732223653469; forceNewPlan=false, retain=false 2024-11-21T21:14:46,033 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=0357b2a5ad6dd1cf7bba0f542d57baae, regionState=OPENING, regionLocation=849dee24a242,41213,1732223653469 2024-11-21T21:14:46,036 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0357b2a5ad6dd1cf7bba0f542d57baae, ASSIGN because future has completed 2024-11-21T21:14:46,036 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469}] 2024-11-21T21:14:46,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-21T21:14:46,196 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:46,196 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} 2024-11-21T21:14:46,197 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:14:46,198 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:14:46,198 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:14:46,198 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:14:46,200 INFO [StoreOpener-0357b2a5ad6dd1cf7bba0f542d57baae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:14:46,202 INFO [StoreOpener-0357b2a5ad6dd1cf7bba0f542d57baae-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T21:14:46,203 INFO [StoreOpener-0357b2a5ad6dd1cf7bba0f542d57baae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0357b2a5ad6dd1cf7bba0f542d57baae columnFamilyName A 2024-11-21T21:14:46,204 DEBUG [StoreOpener-0357b2a5ad6dd1cf7bba0f542d57baae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:46,205 INFO [StoreOpener-0357b2a5ad6dd1cf7bba0f542d57baae-1 {}] regionserver.HStore(327): Store=0357b2a5ad6dd1cf7bba0f542d57baae/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:14:46,205 INFO [StoreOpener-0357b2a5ad6dd1cf7bba0f542d57baae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:14:46,206 INFO [StoreOpener-0357b2a5ad6dd1cf7bba0f542d57baae-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T21:14:46,207 INFO [StoreOpener-0357b2a5ad6dd1cf7bba0f542d57baae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0357b2a5ad6dd1cf7bba0f542d57baae columnFamilyName B 2024-11-21T21:14:46,207 DEBUG [StoreOpener-0357b2a5ad6dd1cf7bba0f542d57baae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:46,208 INFO [StoreOpener-0357b2a5ad6dd1cf7bba0f542d57baae-1 {}] regionserver.HStore(327): Store=0357b2a5ad6dd1cf7bba0f542d57baae/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:14:46,208 INFO [StoreOpener-0357b2a5ad6dd1cf7bba0f542d57baae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:14:46,209 INFO [StoreOpener-0357b2a5ad6dd1cf7bba0f542d57baae-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T21:14:46,210 INFO [StoreOpener-0357b2a5ad6dd1cf7bba0f542d57baae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0357b2a5ad6dd1cf7bba0f542d57baae columnFamilyName C 2024-11-21T21:14:46,210 DEBUG [StoreOpener-0357b2a5ad6dd1cf7bba0f542d57baae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:46,210 INFO [StoreOpener-0357b2a5ad6dd1cf7bba0f542d57baae-1 {}] regionserver.HStore(327): Store=0357b2a5ad6dd1cf7bba0f542d57baae/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:14:46,211 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:46,211 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:14:46,211 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:14:46,212 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:14:46,213 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:14:46,213 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:14:46,214 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T21:14:46,215 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:14:46,217 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T21:14:46,218 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 0357b2a5ad6dd1cf7bba0f542d57baae; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64430984, jitterRate=-0.03990352153778076}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T21:14:46,218 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:14:46,219 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 0357b2a5ad6dd1cf7bba0f542d57baae: Running coprocessor pre-open hook at 1732223686199Writing region info on filesystem at 1732223686199Initializing all the Stores at 1732223686200 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732223686200Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732223686200Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732223686200Cleaning up temporary data from old regions at 1732223686213 (+13 ms)Running coprocessor post-open hooks at 1732223686218 (+5 ms)Region opened successfully at 1732223686219 (+1 ms) 2024-11-21T21:14:46,220 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., pid=34, masterSystemTime=1732223686189 2024-11-21T21:14:46,222 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:46,222 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:46,223 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=0357b2a5ad6dd1cf7bba0f542d57baae, regionState=OPEN, openSeqNum=2, regionLocation=849dee24a242,41213,1732223653469 2024-11-21T21:14:46,225 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 because future has completed 2024-11-21T21:14:46,228 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-21T21:14:46,228 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 in 190 msec 2024-11-21T21:14:46,230 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-21T21:14:46,231 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0357b2a5ad6dd1cf7bba0f542d57baae, ASSIGN in 347 msec 2024-11-21T21:14:46,232 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T21:14:46,232 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732223686232"}]},"ts":"1732223686232"} 2024-11-21T21:14:46,234 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T21:14:46,235 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T21:14:46,237 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1850 sec 2024-11-21T21:14:47,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-21T21:14:47,200 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:14:47,201 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2697ccde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:47,201 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:14:47,202 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:14:47,203 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:14:47,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:14:47,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:14:47,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f024497, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:47,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:14:47,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:14:47,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:14:47,205 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52572, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:14:47,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6999b773, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:47,206 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:14:47,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:14:47,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:14:47,208 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51824, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:14:47,209 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T21:14:47,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 849dee24a242,42485,1732223652547 2024-11-21T21:14:47,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@10957f61 2024-11-21T21:14:47,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T21:14:47,211 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52588, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T21:14:47,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T21:14:47,216 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T21:14:47,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-21T21:14:47,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741974_1150 (size=1097) 2024-11-21T21:14:47,635 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-21T21:14:47,635 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-21T21:14:47,640 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T21:14:47,648 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0357b2a5ad6dd1cf7bba0f542d57baae, REOPEN/MOVE}] 2024-11-21T21:14:47,649 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0357b2a5ad6dd1cf7bba0f542d57baae, REOPEN/MOVE 2024-11-21T21:14:47,651 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=0357b2a5ad6dd1cf7bba0f542d57baae, regionState=CLOSING, regionLocation=849dee24a242,41213,1732223653469 2024-11-21T21:14:47,654 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0357b2a5ad6dd1cf7bba0f542d57baae, REOPEN/MOVE because future has completed 2024-11-21T21:14:47,654 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T21:14:47,654 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469}] 2024-11-21T21:14:47,809 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close 0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:14:47,809 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T21:14:47,809 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing 0357b2a5ad6dd1cf7bba0f542d57baae, disabling compactions & flushes 2024-11-21T21:14:47,809 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:47,809 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:47,809 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. after waiting 0 ms 2024-11-21T21:14:47,809 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:47,813 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T21:14:47,814 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:47,814 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for 0357b2a5ad6dd1cf7bba0f542d57baae: Waiting for close lock at 1732223687809Running coprocessor pre-close hooks at 1732223687809Disabling compacts and flushes for region at 1732223687809Disabling writes for close at 1732223687809Writing region close event to WAL at 1732223687809Running coprocessor post-close hooks at 1732223687814 (+5 ms)Closed at 1732223687814 2024-11-21T21:14:47,814 WARN [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: 0357b2a5ad6dd1cf7bba0f542d57baae to self. 2024-11-21T21:14:47,816 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed 0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:14:47,817 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=0357b2a5ad6dd1cf7bba0f542d57baae, regionState=CLOSED 2024-11-21T21:14:47,818 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 because future has completed 2024-11-21T21:14:47,821 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-21T21:14:47,821 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure 0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 in 165 msec 2024-11-21T21:14:47,822 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0357b2a5ad6dd1cf7bba0f542d57baae, REOPEN/MOVE; state=CLOSED, location=849dee24a242,41213,1732223653469; forceNewPlan=false, retain=true 2024-11-21T21:14:47,973 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=0357b2a5ad6dd1cf7bba0f542d57baae, regionState=OPENING, regionLocation=849dee24a242,41213,1732223653469 2024-11-21T21:14:47,975 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0357b2a5ad6dd1cf7bba0f542d57baae, REOPEN/MOVE because future has completed 2024-11-21T21:14:47,975 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469}] 2024-11-21T21:14:48,130 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:48,131 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} 2024-11-21T21:14:48,131 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:14:48,131 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:14:48,132 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:14:48,132 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:14:48,136 INFO [StoreOpener-0357b2a5ad6dd1cf7bba0f542d57baae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:14:48,137 INFO [StoreOpener-0357b2a5ad6dd1cf7bba0f542d57baae-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T21:14:48,141 INFO [StoreOpener-0357b2a5ad6dd1cf7bba0f542d57baae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0357b2a5ad6dd1cf7bba0f542d57baae columnFamilyName A 2024-11-21T21:14:48,142 DEBUG [StoreOpener-0357b2a5ad6dd1cf7bba0f542d57baae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:48,143 INFO [StoreOpener-0357b2a5ad6dd1cf7bba0f542d57baae-1 {}] regionserver.HStore(327): Store=0357b2a5ad6dd1cf7bba0f542d57baae/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:14:48,143 INFO [StoreOpener-0357b2a5ad6dd1cf7bba0f542d57baae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:14:48,144 INFO [StoreOpener-0357b2a5ad6dd1cf7bba0f542d57baae-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T21:14:48,144 INFO [StoreOpener-0357b2a5ad6dd1cf7bba0f542d57baae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0357b2a5ad6dd1cf7bba0f542d57baae columnFamilyName B 2024-11-21T21:14:48,144 DEBUG [StoreOpener-0357b2a5ad6dd1cf7bba0f542d57baae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:48,145 INFO [StoreOpener-0357b2a5ad6dd1cf7bba0f542d57baae-1 {}] regionserver.HStore(327): Store=0357b2a5ad6dd1cf7bba0f542d57baae/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:14:48,145 INFO [StoreOpener-0357b2a5ad6dd1cf7bba0f542d57baae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:14:48,145 INFO [StoreOpener-0357b2a5ad6dd1cf7bba0f542d57baae-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T21:14:48,145 INFO [StoreOpener-0357b2a5ad6dd1cf7bba0f542d57baae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0357b2a5ad6dd1cf7bba0f542d57baae columnFamilyName C 2024-11-21T21:14:48,145 DEBUG [StoreOpener-0357b2a5ad6dd1cf7bba0f542d57baae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:48,146 INFO [StoreOpener-0357b2a5ad6dd1cf7bba0f542d57baae-1 {}] regionserver.HStore(327): Store=0357b2a5ad6dd1cf7bba0f542d57baae/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:14:48,146 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:48,146 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:14:48,147 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:14:48,148 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:14:48,149 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:14:48,149 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:14:48,149 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T21:14:48,150 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:14:48,151 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 0357b2a5ad6dd1cf7bba0f542d57baae; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69799737, jitterRate=0.040097132325172424}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T21:14:48,151 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:14:48,152 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 0357b2a5ad6dd1cf7bba0f542d57baae: Running coprocessor pre-open hook at 1732223688132Writing region info on filesystem at 1732223688132Initializing all the Stores at 1732223688133 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732223688133Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732223688133Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732223688133Cleaning up temporary data from old regions at 1732223688149 (+16 ms)Running coprocessor post-open hooks at 1732223688151 (+2 ms)Region opened successfully at 1732223688152 (+1 ms) 2024-11-21T21:14:48,152 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., pid=39, masterSystemTime=1732223688127 2024-11-21T21:14:48,154 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:48,155 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:48,155 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=0357b2a5ad6dd1cf7bba0f542d57baae, regionState=OPEN, openSeqNum=5, regionLocation=849dee24a242,41213,1732223653469 2024-11-21T21:14:48,157 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 because future has completed 2024-11-21T21:14:48,161 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-11-21T21:14:48,161 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure 0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 in 183 msec 2024-11-21T21:14:48,163 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-21T21:14:48,163 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0357b2a5ad6dd1cf7bba0f542d57baae, REOPEN/MOVE in 513 msec 2024-11-21T21:14:48,168 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-21T21:14:48,168 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 525 msec 2024-11-21T21:14:48,171 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 951 msec 2024-11-21T21:14:48,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-21T21:14:48,173 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:14:48,174 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cb008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:48,174 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:14:48,174 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:14:48,175 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:14:48,175 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:14:48,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:14:48,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ea8e2cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:48,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:14:48,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:14:48,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:14:48,177 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52608, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:14:48,178 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61643f5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:48,178 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35409f72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:48,178 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:14:48,178 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:14:48,179 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:14:48,179 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:14:48,179 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:14:48,179 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9778420, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:48,179 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:14:48,180 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:14:48,180 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:14:48,181 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52618, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:14:48,181 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5000fc2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:48,182 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7944f9cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:48,182 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:14:48,182 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:14:48,183 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:14:48,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:14:48,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:14:48,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1261462d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:48,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:14:48,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:14:48,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:14:48,184 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52636, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:14:48,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60fd994, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:48,186 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38720fa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:48,186 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:14:48,186 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:14:48,187 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:14:48,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:14:48,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:14:48,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@113690c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:48,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:14:48,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:14:48,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:14:48,188 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52648, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:14:48,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b94659f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:48,189 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c77b203, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:48,189 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:14:48,189 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:14:48,190 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:14:48,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:14:48,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:14:48,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d41038c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:48,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:14:48,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:14:48,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:14:48,191 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52662, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:14:48,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e528586, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:48,193 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e3f6eb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:48,193 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:14:48,193 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:14:48,194 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:14:48,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:14:48,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:14:48,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69861a14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:48,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:14:48,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:14:48,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:14:48,195 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52688, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:14:48,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@580252bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:48,196 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40353459, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:48,196 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:14:48,196 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:14:48,197 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:14:48,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:14:48,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:14:48,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25baa411, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:48,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:14:48,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:14:48,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:14:48,198 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52698, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:14:48,198 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9dbb964, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:48,199 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@160383af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:48,199 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:14:48,199 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:14:48,200 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:14:48,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:14:48,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:14:48,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@287bd40a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:48,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:14:48,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:14:48,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:14:48,201 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52708, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:14:48,202 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6250b33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:48,203 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@123a694c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:48,203 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:14:48,203 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:14:48,204 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:14:48,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:14:48,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:14:48,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1daa546, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:48,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:14:48,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:14:48,205 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:14:48,205 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52738, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:14:48,206 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@579718f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:14:48,207 DEBUG [Thread-736 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:14:48,207 DEBUG [Thread-749 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:14:48,207 DEBUG [Thread-740 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:14:48,207 DEBUG [Thread-743 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:14:48,208 DEBUG [Thread-745 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:14:48,208 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:14:48,208 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:14:48,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:14:48,209 DEBUG [Thread-740 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:14:48,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:14:48,209 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:14:48,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:14:48,209 DEBUG [Thread-736 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:14:48,209 DEBUG [Thread-740 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:14:48,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:14:48,209 DEBUG [Thread-738 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:14:48,210 DEBUG [Thread-732 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:14:48,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-21T21:14:48,210 DEBUG [Thread-747 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:14:48,210 DEBUG [Thread-738 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:14:48,211 DEBUG [Thread-734 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:14:48,211 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:14:48,211 DEBUG [Thread-738 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:14:48,211 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51858, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:14:48,211 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51842, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:14:48,211 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:14:48,211 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51840, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:14:48,211 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51862, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:14:48,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:14:48,212 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:14:48,212 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51890, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:14:48,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:14:48,212 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51874, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:14:48,212 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:14:48,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:14:48,213 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:14:48,213 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51896, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:14:48,213 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:14:48,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T21:14:48,213 DEBUG [Thread-736 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:14:48,214 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:14:48,214 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51920, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:14:48,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5] 2024-11-21T21:14:48,219 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5] 2024-11-21T21:14:48,219 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5] 2024-11-21T21:14:48,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5] 2024-11-21T21:14:48,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5] 2024-11-21T21:14:48,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5] 2024-11-21T21:14:48,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5] 2024-11-21T21:14:48,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5] 2024-11-21T21:14:48,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5] 2024-11-21T21:14:48,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:14:48,231 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0357b2a5ad6dd1cf7bba0f542d57baae 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T21:14:48,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0357b2a5ad6dd1cf7bba0f542d57baae, store=A 2024-11-21T21:14:48,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:48,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0357b2a5ad6dd1cf7bba0f542d57baae, store=B 2024-11-21T21:14:48,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:48,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0357b2a5ad6dd1cf7bba0f542d57baae, store=C 2024-11-21T21:14:48,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:48,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411219d77811cf81148b489be6dc79a43d483_0357b2a5ad6dd1cf7bba0f542d57baae is 50, key is test_row_0/A:col10/1732223688229/Put/seqid=0 2024-11-21T21:14:48,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741975_1151 (size=19474) 2024-11-21T21:14:48,292 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:48,298 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732223688278, max timestamp=1732223688293Num uniques 859; 2024-11-21T21:14:48,298 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T21:14:48,298 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732223688278, max timestamp=1732223688293Num uniques 859; 2024-11-21T21:14:48,298 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T21:14:48,302 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411219d77811cf81148b489be6dc79a43d483_0357b2a5ad6dd1cf7bba0f542d57baae to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219d77811cf81148b489be6dc79a43d483_0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:14:48,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732223688278, max timestamp=1732223688293Num uniques 859; 2024-11-21T21:14:48,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T21:14:48,303 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/aaaf623a271042bab6dbfa087fb5872c, store: [table=TestAcidGuarantees family=A region=0357b2a5ad6dd1cf7bba0f542d57baae] 2024-11-21T21:14:48,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/aaaf623a271042bab6dbfa087fb5872c is 175, key is test_row_0/A:col10/1732223688229/Put/seqid=0 2024-11-21T21:14:48,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T21:14:48,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741976_1152 (size=56733) 2024-11-21T21:14:48,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T21:14:48,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:48,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. as already flushing 2024-11-21T21:14:48,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:48,366 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:48,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:48,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:48,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:48,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51920 deadline: 1732223748374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:48,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:48,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:48,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:48,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51874 deadline: 1732223748374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:48,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:48,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:48,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:48,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:48,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:48,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51840 deadline: 1732223748374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:48,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:48,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:48,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:48,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:48,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1732223748375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:48,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:48,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:48,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:48,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:48,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223748383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:48,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:48,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:48,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:48,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:48,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51920 deadline: 1732223748491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:48,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:48,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51840 deadline: 1732223748491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:48,496 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:48,496 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:48,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:48,496 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:48,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1732223748491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:48,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:48,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:48,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:48,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:48,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:48,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:48,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:48,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51874 deadline: 1732223748491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:48,497 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:48,497 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:48,497 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:48,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:48,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223748501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:48,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:48,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:48,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:48,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T21:14:48,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:48,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. as already flushing 2024-11-21T21:14:48,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:48,520 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:48,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:48,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:48,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T21:14:48,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:48,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51840 deadline: 1732223748701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:48,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:48,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:48,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:48,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51920 deadline: 1732223748701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:48,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:48,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:48,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1732223748702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:48,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:48,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:48,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:48,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:48,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51874 deadline: 1732223748702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:48,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:48,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:48,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:48,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:48,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:48,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:48,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:48,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223748711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:48,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:48,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:48,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:48,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T21:14:48,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:48,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. as already flushing 2024-11-21T21:14:48,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:48,724 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:48,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:48,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:48,731 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/aaaf623a271042bab6dbfa087fb5872c 2024-11-21T21:14:48,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/b4ba571104aa4047bb122ad27ef72bdc is 50, key is test_row_0/B:col10/1732223688229/Put/seqid=0 2024-11-21T21:14:48,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741977_1153 (size=12001) 2024-11-21T21:14:48,790 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/b4ba571104aa4047bb122ad27ef72bdc 2024-11-21T21:14:48,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/a0db3d97b2fa4ef2a44b3c119dd5d6f6 is 50, key is test_row_0/C:col10/1732223688229/Put/seqid=0 2024-11-21T21:14:48,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741978_1154 (size=12001) 2024-11-21T21:14:48,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T21:14:48,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T21:14:48,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:48,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. as already flushing 2024-11-21T21:14:48,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:48,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:48,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:48,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:49,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:49,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:49,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51874 deadline: 1732223749011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:49,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51920 deadline: 1732223749011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:49,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:49,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:49,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51840 deadline: 1732223749011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:49,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:49,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:49,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:49,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1732223749011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:49,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:49,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:49,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:49,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:49,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:49,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:49,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:49,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:49,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:49,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:49,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223749020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:49,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:49,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:49,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:49,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T21:14:49,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:49,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. as already flushing 2024-11-21T21:14:49,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:49,033 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:49,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:49,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:49,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T21:14:49,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:49,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. as already flushing 2024-11-21T21:14:49,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:49,188 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:49,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:49,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:49,239 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/a0db3d97b2fa4ef2a44b3c119dd5d6f6 2024-11-21T21:14:49,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/aaaf623a271042bab6dbfa087fb5872c as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/aaaf623a271042bab6dbfa087fb5872c 2024-11-21T21:14:49,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/aaaf623a271042bab6dbfa087fb5872c, entries=300, sequenceid=14, filesize=55.4 K 2024-11-21T21:14:49,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/b4ba571104aa4047bb122ad27ef72bdc as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/b4ba571104aa4047bb122ad27ef72bdc 2024-11-21T21:14:49,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/b4ba571104aa4047bb122ad27ef72bdc, entries=150, sequenceid=14, filesize=11.7 K 2024-11-21T21:14:49,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/a0db3d97b2fa4ef2a44b3c119dd5d6f6 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/a0db3d97b2fa4ef2a44b3c119dd5d6f6 2024-11-21T21:14:49,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/a0db3d97b2fa4ef2a44b3c119dd5d6f6, entries=150, sequenceid=14, filesize=11.7 K 2024-11-21T21:14:49,274 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 0357b2a5ad6dd1cf7bba0f542d57baae in 1043ms, sequenceid=14, compaction requested=false 2024-11-21T21:14:49,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0357b2a5ad6dd1cf7bba0f542d57baae: 2024-11-21T21:14:49,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T21:14:49,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:49,341 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 0357b2a5ad6dd1cf7bba0f542d57baae 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-21T21:14:49,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0357b2a5ad6dd1cf7bba0f542d57baae, store=A 2024-11-21T21:14:49,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:14:49,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0357b2a5ad6dd1cf7bba0f542d57baae, store=B 2024-11-21T21:14:49,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:14:49,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0357b2a5ad6dd1cf7bba0f542d57baae, store=C 2024-11-21T21:14:49,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:14:49,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T21:14:49,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ab19cd5a0d904dd7aedffa91b034c183_0357b2a5ad6dd1cf7bba0f542d57baae is 50, key is test_row_0/A:col10/1732223688373/Put/seqid=0 2024-11-21T21:14:49,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741979_1155 (size=12154) 2024-11-21T21:14:49,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:49,380 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ab19cd5a0d904dd7aedffa91b034c183_0357b2a5ad6dd1cf7bba0f542d57baae to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ab19cd5a0d904dd7aedffa91b034c183_0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:14:49,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/37f5a2ac1c0f4a8593de2fb4730ae978, store: [table=TestAcidGuarantees family=A region=0357b2a5ad6dd1cf7bba0f542d57baae] 2024-11-21T21:14:49,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/37f5a2ac1c0f4a8593de2fb4730ae978 is 175, key is test_row_0/A:col10/1732223688373/Put/seqid=0 2024-11-21T21:14:49,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741980_1156 (size=30955) 2024-11-21T21:14:49,400 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/37f5a2ac1c0f4a8593de2fb4730ae978 2024-11-21T21:14:49,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/87526025ca074be0882e55cea07fc2a8 is 50, key is test_row_0/B:col10/1732223688373/Put/seqid=0 2024-11-21T21:14:49,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741981_1157 (size=12001) 2024-11-21T21:14:49,426 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/87526025ca074be0882e55cea07fc2a8 2024-11-21T21:14:49,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/3ab1ed8067c0434d951ac2712ff059e6 is 50, key is test_row_0/C:col10/1732223688373/Put/seqid=0 2024-11-21T21:14:49,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741982_1158 (size=12001) 2024-11-21T21:14:49,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:14:49,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. as already flushing 2024-11-21T21:14:49,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:49,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51840 deadline: 1732223749537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:49,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:49,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51874 deadline: 1732223749538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:49,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:49,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51920 deadline: 1732223749540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:49,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:49,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:49,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:49,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:49,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:49,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223749540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:49,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:49,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:49,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:49,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:49,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:49,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:49,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:49,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:49,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:49,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1732223749541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:49,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:49,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:49,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:49,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:49,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223749651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:49,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:49,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:49,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1732223749651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:49,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:49,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:49,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:49,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:49,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:49,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51840 deadline: 1732223749652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:49,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:49,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:49,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:49,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:49,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:49,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51920 deadline: 1732223749652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:49,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:49,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:49,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:49,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:49,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51874 deadline: 1732223749652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:49,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:49,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:49,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:49,856 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/3ab1ed8067c0434d951ac2712ff059e6 2024-11-21T21:14:49,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/37f5a2ac1c0f4a8593de2fb4730ae978 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/37f5a2ac1c0f4a8593de2fb4730ae978 2024-11-21T21:14:49,867 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/37f5a2ac1c0f4a8593de2fb4730ae978, entries=150, sequenceid=51, filesize=30.2 K 2024-11-21T21:14:49,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/87526025ca074be0882e55cea07fc2a8 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/87526025ca074be0882e55cea07fc2a8 2024-11-21T21:14:49,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:49,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:49,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1732223749870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:49,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223749870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:49,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:49,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:49,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:49,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:49,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:49,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:49,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:49,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51840 deadline: 1732223749870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:49,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:49,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51920 deadline: 1732223749870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:49,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:49,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:49,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:49,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:49,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:49,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:49,876 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/87526025ca074be0882e55cea07fc2a8, entries=150, sequenceid=51, filesize=11.7 K 2024-11-21T21:14:49,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:49,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51874 deadline: 1732223749870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:49,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:49,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:49,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:49,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/3ab1ed8067c0434d951ac2712ff059e6 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/3ab1ed8067c0434d951ac2712ff059e6 2024-11-21T21:14:49,882 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/3ab1ed8067c0434d951ac2712ff059e6, entries=150, sequenceid=51, filesize=11.7 K 2024-11-21T21:14:49,884 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 0357b2a5ad6dd1cf7bba0f542d57baae in 543ms, sequenceid=51, compaction requested=false 2024-11-21T21:14:49,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 0357b2a5ad6dd1cf7bba0f542d57baae: 2024-11-21T21:14:49,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:49,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-21T21:14:49,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-21T21:14:49,889 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-21T21:14:49,890 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6730 sec 2024-11-21T21:14:49,893 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.6830 sec 2024-11-21T21:14:50,183 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0357b2a5ad6dd1cf7bba0f542d57baae 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-21T21:14:50,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:14:50,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0357b2a5ad6dd1cf7bba0f542d57baae, store=A 2024-11-21T21:14:50,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:50,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0357b2a5ad6dd1cf7bba0f542d57baae, store=B 2024-11-21T21:14:50,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:50,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0357b2a5ad6dd1cf7bba0f542d57baae, store=C 2024-11-21T21:14:50,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:50,188 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T21:14:50,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c6085d10aebe477b8c056b070c12f5f5_0357b2a5ad6dd1cf7bba0f542d57baae is 50, key is test_row_0/A:col10/1732223690183/Put/seqid=0 2024-11-21T21:14:50,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741983_1159 (size=12154) 2024-11-21T21:14:50,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:50,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1732223750214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:50,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:50,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51920 deadline: 1732223750214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:50,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:50,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:50,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:50,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:50,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:50,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:50,225 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:50,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:50,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223750222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:50,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:50,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:50,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:50,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:50,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51874 deadline: 1732223750223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:50,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:50,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:50,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:50,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:50,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51840 deadline: 1732223750224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:50,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:50,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:50,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:50,230 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c6085d10aebe477b8c056b070c12f5f5_0357b2a5ad6dd1cf7bba0f542d57baae to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c6085d10aebe477b8c056b070c12f5f5_0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:14:50,241 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/bc88da1a9324414193c161815e5bf91f, store: [table=TestAcidGuarantees family=A region=0357b2a5ad6dd1cf7bba0f542d57baae] 2024-11-21T21:14:50,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/bc88da1a9324414193c161815e5bf91f is 175, key is test_row_0/A:col10/1732223690183/Put/seqid=0 2024-11-21T21:14:50,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741984_1160 (size=30955) 2024-11-21T21:14:50,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:50,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51920 deadline: 1732223750330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:50,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:50,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:50,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:50,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:50,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223750331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:50,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:50,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:50,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:50,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:50,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1732223750331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:50,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:50,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:50,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:50,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:50,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51874 deadline: 1732223750340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:50,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:50,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:50,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:50,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:50,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51840 deadline: 1732223750340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:50,345 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:50,346 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:50,346 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:50,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T21:14:50,361 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:14:50,362 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:14:50,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-21T21:14:50,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T21:14:50,366 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:14:50,367 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:14:50,367 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:14:50,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T21:14:50,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-21T21:14:50,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:50,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. as already flushing 2024-11-21T21:14:50,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:50,520 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:50,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:50,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:50,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:50,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1732223750541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:50,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:50,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51920 deadline: 1732223750541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:50,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:50,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223750541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:50,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:50,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:50,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:50,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:50,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:50,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:50,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:50,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:50,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:50,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:50,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51874 deadline: 1732223750550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:50,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:50,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51840 deadline: 1732223750550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:50,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:50,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:50,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:50,553 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:50,553 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:50,553 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:50,650 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=64, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/bc88da1a9324414193c161815e5bf91f 2024-11-21T21:14:50,660 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/fdde0fdfcf034e7fb2d03714815ef40d is 50, key is test_row_0/B:col10/1732223690183/Put/seqid=0 2024-11-21T21:14:50,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741985_1161 (size=12001) 2024-11-21T21:14:50,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-21T21:14:50,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:50,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. as already flushing 2024-11-21T21:14:50,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:50,674 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:50,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:50,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:50,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T21:14:50,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-21T21:14:50,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:50,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. as already flushing 2024-11-21T21:14:50,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:50,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:50,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:50,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:50,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:50,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1732223750850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:50,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:50,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:50,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51920 deadline: 1732223750851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:50,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:50,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:50,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:50,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:50,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:50,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223750851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:50,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:50,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:50,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:50,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:50,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:50,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51840 deadline: 1732223750860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:50,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:50,861 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:50,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51874 deadline: 1732223750860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:50,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:50,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:50,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:50,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:50,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:50,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-21T21:14:50,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:50,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. as already flushing 2024-11-21T21:14:50,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:50,980 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:50,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:50,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:50,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T21:14:51,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/fdde0fdfcf034e7fb2d03714815ef40d 2024-11-21T21:14:51,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/600c72b442eb4b84b5229905d93a414a is 50, key is test_row_0/C:col10/1732223690183/Put/seqid=0 2024-11-21T21:14:51,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741986_1162 (size=12001) 2024-11-21T21:14:51,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-21T21:14:51,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:51,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. as already flushing 2024-11-21T21:14:51,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:51,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:51,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:51,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:51,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-21T21:14:51,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:51,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. as already flushing 2024-11-21T21:14:51,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:51,285 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:51,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:51,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:51,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:51,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1732223751360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:51,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:51,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51920 deadline: 1732223751360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:51,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:51,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:51,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:51,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:51,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223751360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:51,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:51,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:51,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:51,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:51,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:51,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:51,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:51,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51840 deadline: 1732223751370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:51,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:51,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:51,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:51,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:51,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51874 deadline: 1732223751370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:51,373 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:51,373 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:51,373 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:51,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-21T21:14:51,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:51,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. as already flushing 2024-11-21T21:14:51,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:51,439 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:51,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:51,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:51,485 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/600c72b442eb4b84b5229905d93a414a 2024-11-21T21:14:51,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/bc88da1a9324414193c161815e5bf91f as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/bc88da1a9324414193c161815e5bf91f 2024-11-21T21:14:51,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/bc88da1a9324414193c161815e5bf91f, entries=150, sequenceid=64, filesize=30.2 K 2024-11-21T21:14:51,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/fdde0fdfcf034e7fb2d03714815ef40d as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/fdde0fdfcf034e7fb2d03714815ef40d 2024-11-21T21:14:51,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T21:14:51,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/fdde0fdfcf034e7fb2d03714815ef40d, entries=150, sequenceid=64, filesize=11.7 K 2024-11-21T21:14:51,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/600c72b442eb4b84b5229905d93a414a as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/600c72b442eb4b84b5229905d93a414a 2024-11-21T21:14:51,509 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/600c72b442eb4b84b5229905d93a414a, entries=150, sequenceid=64, filesize=11.7 K 2024-11-21T21:14:51,510 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 0357b2a5ad6dd1cf7bba0f542d57baae in 1328ms, sequenceid=64, compaction requested=true 2024-11-21T21:14:51,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0357b2a5ad6dd1cf7bba0f542d57baae: 2024-11-21T21:14:51,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0357b2a5ad6dd1cf7bba0f542d57baae:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:14:51,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:51,510 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:14:51,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0357b2a5ad6dd1cf7bba0f542d57baae:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:14:51,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:51,510 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:14:51,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0357b2a5ad6dd1cf7bba0f542d57baae:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:14:51,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:14:51,511 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:14:51,511 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:14:51,512 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): 0357b2a5ad6dd1cf7bba0f542d57baae/A is initiating minor compaction (all files) 2024-11-21T21:14:51,512 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): 0357b2a5ad6dd1cf7bba0f542d57baae/B is initiating minor compaction (all files) 2024-11-21T21:14:51,512 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0357b2a5ad6dd1cf7bba0f542d57baae/B in TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:51,512 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0357b2a5ad6dd1cf7bba0f542d57baae/A in TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:51,512 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/b4ba571104aa4047bb122ad27ef72bdc, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/87526025ca074be0882e55cea07fc2a8, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/fdde0fdfcf034e7fb2d03714815ef40d] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp, totalSize=35.2 K 2024-11-21T21:14:51,512 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/aaaf623a271042bab6dbfa087fb5872c, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/37f5a2ac1c0f4a8593de2fb4730ae978, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/bc88da1a9324414193c161815e5bf91f] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp, totalSize=115.9 K 2024-11-21T21:14:51,512 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:51,512 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. files: [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/aaaf623a271042bab6dbfa087fb5872c, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/37f5a2ac1c0f4a8593de2fb4730ae978, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/bc88da1a9324414193c161815e5bf91f] 2024-11-21T21:14:51,512 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting aaaf623a271042bab6dbfa087fb5872c, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732223688223 2024-11-21T21:14:51,512 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting b4ba571104aa4047bb122ad27ef72bdc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732223688227 2024-11-21T21:14:51,513 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37f5a2ac1c0f4a8593de2fb4730ae978, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732223688309 2024-11-21T21:14:51,513 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 87526025ca074be0882e55cea07fc2a8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732223688309 2024-11-21T21:14:51,513 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc88da1a9324414193c161815e5bf91f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732223689537 2024-11-21T21:14:51,513 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting fdde0fdfcf034e7fb2d03714815ef40d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732223689537 2024-11-21T21:14:51,519 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0357b2a5ad6dd1cf7bba0f542d57baae] 2024-11-21T21:14:51,522 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121f5a21fd30e474a3a8eab8eb9805fe5ca_0357b2a5ad6dd1cf7bba0f542d57baae store=[table=TestAcidGuarantees family=A region=0357b2a5ad6dd1cf7bba0f542d57baae] 2024-11-21T21:14:51,522 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0357b2a5ad6dd1cf7bba0f542d57baae#B#compaction#145 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:14:51,522 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/4fb93c62d4fb40dd97b51d5a34dcec8e is 50, key is test_row_0/B:col10/1732223690183/Put/seqid=0 2024-11-21T21:14:51,529 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121f5a21fd30e474a3a8eab8eb9805fe5ca_0357b2a5ad6dd1cf7bba0f542d57baae, store=[table=TestAcidGuarantees family=A region=0357b2a5ad6dd1cf7bba0f542d57baae] 2024-11-21T21:14:51,529 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121f5a21fd30e474a3a8eab8eb9805fe5ca_0357b2a5ad6dd1cf7bba0f542d57baae because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0357b2a5ad6dd1cf7bba0f542d57baae] 2024-11-21T21:14:51,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741987_1163 (size=12104) 2024-11-21T21:14:51,542 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/4fb93c62d4fb40dd97b51d5a34dcec8e as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/4fb93c62d4fb40dd97b51d5a34dcec8e 2024-11-21T21:14:51,552 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0357b2a5ad6dd1cf7bba0f542d57baae/B of 0357b2a5ad6dd1cf7bba0f542d57baae into 4fb93c62d4fb40dd97b51d5a34dcec8e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:14:51,552 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0357b2a5ad6dd1cf7bba0f542d57baae: 2024-11-21T21:14:51,552 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., storeName=0357b2a5ad6dd1cf7bba0f542d57baae/B, priority=13, startTime=1732223691510; duration=0sec 2024-11-21T21:14:51,552 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:14:51,554 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0357b2a5ad6dd1cf7bba0f542d57baae:B 2024-11-21T21:14:51,554 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:14:51,556 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:14:51,556 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): 0357b2a5ad6dd1cf7bba0f542d57baae/C is initiating minor compaction (all files) 2024-11-21T21:14:51,556 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0357b2a5ad6dd1cf7bba0f542d57baae/C in TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:51,556 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/a0db3d97b2fa4ef2a44b3c119dd5d6f6, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/3ab1ed8067c0434d951ac2712ff059e6, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/600c72b442eb4b84b5229905d93a414a] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp, totalSize=35.2 K 2024-11-21T21:14:51,556 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting a0db3d97b2fa4ef2a44b3c119dd5d6f6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732223688227 2024-11-21T21:14:51,557 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ab1ed8067c0434d951ac2712ff059e6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732223688309 2024-11-21T21:14:51,557 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 600c72b442eb4b84b5229905d93a414a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732223689537 2024-11-21T21:14:51,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741988_1164 (size=4469) 2024-11-21T21:14:51,563 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0357b2a5ad6dd1cf7bba0f542d57baae#A#compaction#144 average throughput is 0.56 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:14:51,563 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/507143321ca44947aa9232c2d8538355 is 175, key is test_row_0/A:col10/1732223690183/Put/seqid=0 2024-11-21T21:14:51,573 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0357b2a5ad6dd1cf7bba0f542d57baae#C#compaction#146 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:14:51,574 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/1d322b6eaa16455590f35a46d059109f is 50, key is test_row_0/C:col10/1732223690183/Put/seqid=0 2024-11-21T21:14:51,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741989_1165 (size=31058) 2024-11-21T21:14:51,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-21T21:14:51,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:51,592 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 0357b2a5ad6dd1cf7bba0f542d57baae 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-21T21:14:51,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0357b2a5ad6dd1cf7bba0f542d57baae, store=A 2024-11-21T21:14:51,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:51,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0357b2a5ad6dd1cf7bba0f542d57baae, store=B 2024-11-21T21:14:51,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:51,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0357b2a5ad6dd1cf7bba0f542d57baae, store=C 2024-11-21T21:14:51,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:51,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741990_1166 (size=12104) 2024-11-21T21:14:51,602 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/1d322b6eaa16455590f35a46d059109f as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/1d322b6eaa16455590f35a46d059109f 2024-11-21T21:14:51,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112156543518382c47fd91c37e260399b2ed_0357b2a5ad6dd1cf7bba0f542d57baae is 50, key is test_row_0/A:col10/1732223690205/Put/seqid=0 2024-11-21T21:14:51,607 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0357b2a5ad6dd1cf7bba0f542d57baae/C of 0357b2a5ad6dd1cf7bba0f542d57baae into 1d322b6eaa16455590f35a46d059109f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:14:51,607 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0357b2a5ad6dd1cf7bba0f542d57baae: 2024-11-21T21:14:51,607 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., storeName=0357b2a5ad6dd1cf7bba0f542d57baae/C, priority=13, startTime=1732223691510; duration=0sec 2024-11-21T21:14:51,608 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:51,608 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0357b2a5ad6dd1cf7bba0f542d57baae:C 2024-11-21T21:14:51,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741991_1167 (size=12154) 2024-11-21T21:14:51,984 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/507143321ca44947aa9232c2d8538355 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/507143321ca44947aa9232c2d8538355 2024-11-21T21:14:51,990 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0357b2a5ad6dd1cf7bba0f542d57baae/A of 0357b2a5ad6dd1cf7bba0f542d57baae into 507143321ca44947aa9232c2d8538355(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:14:51,990 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0357b2a5ad6dd1cf7bba0f542d57baae: 2024-11-21T21:14:51,990 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., storeName=0357b2a5ad6dd1cf7bba0f542d57baae/A, priority=13, startTime=1732223691510; duration=0sec 2024-11-21T21:14:51,991 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:51,991 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0357b2a5ad6dd1cf7bba0f542d57baae:A 2024-11-21T21:14:52,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:52,016 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112156543518382c47fd91c37e260399b2ed_0357b2a5ad6dd1cf7bba0f542d57baae to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112156543518382c47fd91c37e260399b2ed_0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:14:52,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/ca1e1974896f492ea4d8ebb6c41e6a2e, store: [table=TestAcidGuarantees family=A region=0357b2a5ad6dd1cf7bba0f542d57baae] 2024-11-21T21:14:52,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/ca1e1974896f492ea4d8ebb6c41e6a2e is 175, key is test_row_0/A:col10/1732223690205/Put/seqid=0 2024-11-21T21:14:52,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741992_1168 (size=30955) 2024-11-21T21:14:52,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:14:52,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. as already flushing 2024-11-21T21:14:52,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:52,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1732223752388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:52,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:52,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223752389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:52,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:52,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:52,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:52,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:52,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:52,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:52,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:52,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51920 deadline: 1732223752393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:52,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:52,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:52,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51874 deadline: 1732223752395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:52,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51840 deadline: 1732223752395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:52,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:52,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:52,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:52,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:52,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:52,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:52,403 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:52,403 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:52,403 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:52,422 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=84, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/ca1e1974896f492ea4d8ebb6c41e6a2e 2024-11-21T21:14:52,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/8144e25ce95c4f7d8d1cb5940adad345 is 50, key is test_row_0/B:col10/1732223690205/Put/seqid=0 2024-11-21T21:14:52,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741993_1169 (size=12001) 2024-11-21T21:14:52,456 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/8144e25ce95c4f7d8d1cb5940adad345 2024-11-21T21:14:52,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/c428e4d5eeed47998cc8e75ae4b1e874 is 50, key is test_row_0/C:col10/1732223690205/Put/seqid=0 2024-11-21T21:14:52,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741994_1170 (size=12001) 2024-11-21T21:14:52,481 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/c428e4d5eeed47998cc8e75ae4b1e874 2024-11-21T21:14:52,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/ca1e1974896f492ea4d8ebb6c41e6a2e as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/ca1e1974896f492ea4d8ebb6c41e6a2e 2024-11-21T21:14:52,495 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/ca1e1974896f492ea4d8ebb6c41e6a2e, entries=150, sequenceid=84, filesize=30.2 K 2024-11-21T21:14:52,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/8144e25ce95c4f7d8d1cb5940adad345 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/8144e25ce95c4f7d8d1cb5940adad345 2024-11-21T21:14:52,504 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/8144e25ce95c4f7d8d1cb5940adad345, entries=150, sequenceid=84, filesize=11.7 K 2024-11-21T21:14:52,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/c428e4d5eeed47998cc8e75ae4b1e874 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/c428e4d5eeed47998cc8e75ae4b1e874 2024-11-21T21:14:52,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:52,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223752500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:52,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1732223752500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T21:14:52,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:52,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:52,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:52,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:52,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:52,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:52,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51840 deadline: 1732223752511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:52,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51920 deadline: 1732223752511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:52,516 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:52,516 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:52,516 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:52,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51874 deadline: 1732223752511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:52,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:52,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:52,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:52,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:52,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:52,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:52,523 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/c428e4d5eeed47998cc8e75ae4b1e874, entries=150, sequenceid=84, filesize=11.7 K 2024-11-21T21:14:52,524 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 0357b2a5ad6dd1cf7bba0f542d57baae in 933ms, sequenceid=84, compaction requested=false 2024-11-21T21:14:52,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 0357b2a5ad6dd1cf7bba0f542d57baae: 2024-11-21T21:14:52,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:52,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-21T21:14:52,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-21T21:14:52,527 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-21T21:14:52,527 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1590 sec 2024-11-21T21:14:52,530 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.1660 sec 2024-11-21T21:14:52,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:14:52,723 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0357b2a5ad6dd1cf7bba0f542d57baae 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-21T21:14:52,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0357b2a5ad6dd1cf7bba0f542d57baae, store=A 2024-11-21T21:14:52,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:52,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0357b2a5ad6dd1cf7bba0f542d57baae, store=B 2024-11-21T21:14:52,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:52,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0357b2a5ad6dd1cf7bba0f542d57baae, store=C 2024-11-21T21:14:52,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:52,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112163b22fef24dd4801a4d15d2dbfd77dd1_0357b2a5ad6dd1cf7bba0f542d57baae is 50, key is test_row_0/A:col10/1732223692391/Put/seqid=0 2024-11-21T21:14:52,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741995_1171 (size=12154) 2024-11-21T21:14:52,758 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:52,764 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112163b22fef24dd4801a4d15d2dbfd77dd1_0357b2a5ad6dd1cf7bba0f542d57baae to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112163b22fef24dd4801a4d15d2dbfd77dd1_0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:14:52,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:52,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51874 deadline: 1732223752751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:52,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:52,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:52,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:52,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223752751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:52,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:52,766 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/a859aca6d38146aaa073455f43b94e49, store: [table=TestAcidGuarantees family=A region=0357b2a5ad6dd1cf7bba0f542d57baae] 2024-11-21T21:14:52,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:52,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:52,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:52,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51840 deadline: 1732223752752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:52,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:52,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/a859aca6d38146aaa073455f43b94e49 is 175, key is test_row_0/A:col10/1732223692391/Put/seqid=0 2024-11-21T21:14:52,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:52,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:52,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:52,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:52,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51920 deadline: 1732223752766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:52,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:52,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1732223752768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:52,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:52,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:52,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:52,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:52,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:52,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:52,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741996_1172 (size=30955) 2024-11-21T21:14:52,788 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=100, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/a859aca6d38146aaa073455f43b94e49 2024-11-21T21:14:52,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/7a44c5f86fb44278bf874cf6100b2b5c is 50, key is test_row_0/B:col10/1732223692391/Put/seqid=0 2024-11-21T21:14:52,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741997_1173 (size=12001) 2024-11-21T21:14:52,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:52,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51840 deadline: 1732223752870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:52,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:52,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:52,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:52,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:52,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223752871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:52,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:52,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51874 deadline: 1732223752870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:52,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:52,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1732223752881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:52,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:52,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:52,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:52,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:52,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51920 deadline: 1732223752881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:52,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:52,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:52,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:53,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:53,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51874 deadline: 1732223753081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:53,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:53,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223753082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:53,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:53,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51840 deadline: 1732223753084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:53,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:53,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:53,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:53,088 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:53,088 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:53,088 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:53,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:53,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:53,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:53,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:53,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1732223753090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:53,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:53,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:53,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51920 deadline: 1732223753090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:53,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:53,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:53,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:53,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:53,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:53,211 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/7a44c5f86fb44278bf874cf6100b2b5c 2024-11-21T21:14:53,219 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/b1315c76ddf940138a2b93c4a3e590ae is 50, key is test_row_0/C:col10/1732223692391/Put/seqid=0 2024-11-21T21:14:53,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741998_1174 (size=12001) 2024-11-21T21:14:53,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:53,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51840 deadline: 1732223753391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:53,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:53,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:53,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:53,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:53,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223753400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:53,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:53,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:53,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:53,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51874 deadline: 1732223753400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:53,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:53,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:53,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:53,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:53,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:53,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1732223753400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:53,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:53,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51920 deadline: 1732223753400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:53,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:53,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:53,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:53,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:53,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:53,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:53,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/b1315c76ddf940138a2b93c4a3e590ae 2024-11-21T21:14:53,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/a859aca6d38146aaa073455f43b94e49 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/a859aca6d38146aaa073455f43b94e49 2024-11-21T21:14:53,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/a859aca6d38146aaa073455f43b94e49, entries=150, sequenceid=100, filesize=30.2 K 2024-11-21T21:14:53,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/7a44c5f86fb44278bf874cf6100b2b5c as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/7a44c5f86fb44278bf874cf6100b2b5c 2024-11-21T21:14:53,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/7a44c5f86fb44278bf874cf6100b2b5c, entries=150, sequenceid=100, filesize=11.7 K 2024-11-21T21:14:53,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/b1315c76ddf940138a2b93c4a3e590ae as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/b1315c76ddf940138a2b93c4a3e590ae 2024-11-21T21:14:53,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/b1315c76ddf940138a2b93c4a3e590ae, entries=150, sequenceid=100, filesize=11.7 K 2024-11-21T21:14:53,647 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 0357b2a5ad6dd1cf7bba0f542d57baae in 925ms, sequenceid=100, compaction requested=true 2024-11-21T21:14:53,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0357b2a5ad6dd1cf7bba0f542d57baae: 2024-11-21T21:14:53,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0357b2a5ad6dd1cf7bba0f542d57baae:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:14:53,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:53,647 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:14:53,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0357b2a5ad6dd1cf7bba0f542d57baae:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:14:53,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:53,647 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:14:53,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0357b2a5ad6dd1cf7bba0f542d57baae:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:14:53,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:14:53,649 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:14:53,649 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:14:53,649 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): 0357b2a5ad6dd1cf7bba0f542d57baae/A is initiating minor compaction (all files) 2024-11-21T21:14:53,649 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): 0357b2a5ad6dd1cf7bba0f542d57baae/B is initiating minor compaction (all files) 2024-11-21T21:14:53,649 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0357b2a5ad6dd1cf7bba0f542d57baae/B in TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:53,649 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0357b2a5ad6dd1cf7bba0f542d57baae/A in TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:53,649 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/507143321ca44947aa9232c2d8538355, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/ca1e1974896f492ea4d8ebb6c41e6a2e, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/a859aca6d38146aaa073455f43b94e49] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp, totalSize=90.8 K 2024-11-21T21:14:53,649 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/4fb93c62d4fb40dd97b51d5a34dcec8e, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/8144e25ce95c4f7d8d1cb5940adad345, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/7a44c5f86fb44278bf874cf6100b2b5c] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp, totalSize=35.3 K 2024-11-21T21:14:53,649 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:53,649 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. files: [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/507143321ca44947aa9232c2d8538355, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/ca1e1974896f492ea4d8ebb6c41e6a2e, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/a859aca6d38146aaa073455f43b94e49] 2024-11-21T21:14:53,650 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fb93c62d4fb40dd97b51d5a34dcec8e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732223689537 2024-11-21T21:14:53,650 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 507143321ca44947aa9232c2d8538355, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732223689537 2024-11-21T21:14:53,650 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 8144e25ce95c4f7d8d1cb5940adad345, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732223690205 2024-11-21T21:14:53,650 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca1e1974896f492ea4d8ebb6c41e6a2e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732223690205 2024-11-21T21:14:53,650 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a44c5f86fb44278bf874cf6100b2b5c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732223692391 2024-11-21T21:14:53,650 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting a859aca6d38146aaa073455f43b94e49, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732223692391 2024-11-21T21:14:53,655 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0357b2a5ad6dd1cf7bba0f542d57baae] 2024-11-21T21:14:53,657 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411211de1729149944d0aa1a0846711000f4f_0357b2a5ad6dd1cf7bba0f542d57baae store=[table=TestAcidGuarantees family=A region=0357b2a5ad6dd1cf7bba0f542d57baae] 2024-11-21T21:14:53,659 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0357b2a5ad6dd1cf7bba0f542d57baae#B#compaction#154 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:14:53,659 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/cd9c6020491c4a73ad896307eaac6d43 is 50, key is test_row_0/B:col10/1732223692391/Put/seqid=0 2024-11-21T21:14:53,661 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411211de1729149944d0aa1a0846711000f4f_0357b2a5ad6dd1cf7bba0f542d57baae, store=[table=TestAcidGuarantees family=A region=0357b2a5ad6dd1cf7bba0f542d57baae] 2024-11-21T21:14:53,661 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411211de1729149944d0aa1a0846711000f4f_0357b2a5ad6dd1cf7bba0f542d57baae because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0357b2a5ad6dd1cf7bba0f542d57baae] 2024-11-21T21:14:53,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741999_1175 (size=12207) 2024-11-21T21:14:53,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742000_1176 (size=4469) 2024-11-21T21:14:53,677 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0357b2a5ad6dd1cf7bba0f542d57baae#A#compaction#153 average throughput is 1.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:14:53,678 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/9da84e753020408788b2a0065a1d18e5 is 175, key is test_row_0/A:col10/1732223692391/Put/seqid=0 2024-11-21T21:14:53,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742001_1177 (size=31161) 2024-11-21T21:14:53,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:14:53,902 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0357b2a5ad6dd1cf7bba0f542d57baae 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-21T21:14:53,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0357b2a5ad6dd1cf7bba0f542d57baae, store=A 2024-11-21T21:14:53,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:53,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0357b2a5ad6dd1cf7bba0f542d57baae, store=B 2024-11-21T21:14:53,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:53,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0357b2a5ad6dd1cf7bba0f542d57baae, store=C 2024-11-21T21:14:53,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:53,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411215424bae0684947fe96253dcb6ab7bf49_0357b2a5ad6dd1cf7bba0f542d57baae is 50, key is test_row_0/A:col10/1732223693900/Put/seqid=0 2024-11-21T21:14:53,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742002_1178 (size=14594) 2024-11-21T21:14:53,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:53,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51874 deadline: 1732223753921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:53,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:53,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:53,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:53,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:53,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51920 deadline: 1732223753921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:53,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:53,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:53,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:53,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:53,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51840 deadline: 1732223753924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:53,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:53,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:53,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:53,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:53,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1732223753925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:53,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:53,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:53,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:53,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:53,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223753926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:53,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:53,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:53,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:54,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:54,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51840 deadline: 1732223754040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:54,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:54,042 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:54,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1732223754040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:54,042 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:54,042 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:54,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:54,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:54,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:54,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:54,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223754040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:54,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:54,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:54,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:54,073 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/cd9c6020491c4a73ad896307eaac6d43 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/cd9c6020491c4a73ad896307eaac6d43 2024-11-21T21:14:54,078 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0357b2a5ad6dd1cf7bba0f542d57baae/B of 0357b2a5ad6dd1cf7bba0f542d57baae into cd9c6020491c4a73ad896307eaac6d43(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:14:54,078 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0357b2a5ad6dd1cf7bba0f542d57baae: 2024-11-21T21:14:54,078 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., storeName=0357b2a5ad6dd1cf7bba0f542d57baae/B, priority=13, startTime=1732223693647; duration=0sec 2024-11-21T21:14:54,078 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:14:54,078 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0357b2a5ad6dd1cf7bba0f542d57baae:B 2024-11-21T21:14:54,078 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:14:54,079 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:14:54,079 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): 0357b2a5ad6dd1cf7bba0f542d57baae/C is initiating minor compaction (all files) 2024-11-21T21:14:54,079 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0357b2a5ad6dd1cf7bba0f542d57baae/C in TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:54,079 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/1d322b6eaa16455590f35a46d059109f, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/c428e4d5eeed47998cc8e75ae4b1e874, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/b1315c76ddf940138a2b93c4a3e590ae] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp, totalSize=35.3 K 2024-11-21T21:14:54,080 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d322b6eaa16455590f35a46d059109f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732223689537 2024-11-21T21:14:54,080 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting c428e4d5eeed47998cc8e75ae4b1e874, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732223690205 2024-11-21T21:14:54,080 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting b1315c76ddf940138a2b93c4a3e590ae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732223692391 2024-11-21T21:14:54,086 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/9da84e753020408788b2a0065a1d18e5 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/9da84e753020408788b2a0065a1d18e5 2024-11-21T21:14:54,091 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0357b2a5ad6dd1cf7bba0f542d57baae#C#compaction#156 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:14:54,092 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/ad1a008a471c4807a5fda9e830abf337 is 50, key is test_row_0/C:col10/1732223692391/Put/seqid=0 2024-11-21T21:14:54,095 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0357b2a5ad6dd1cf7bba0f542d57baae/A of 0357b2a5ad6dd1cf7bba0f542d57baae into 9da84e753020408788b2a0065a1d18e5(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:14:54,095 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0357b2a5ad6dd1cf7bba0f542d57baae: 2024-11-21T21:14:54,095 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., storeName=0357b2a5ad6dd1cf7bba0f542d57baae/A, priority=13, startTime=1732223693647; duration=0sec 2024-11-21T21:14:54,095 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:54,096 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0357b2a5ad6dd1cf7bba0f542d57baae:A 2024-11-21T21:14:54,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742003_1179 (size=12207) 2024-11-21T21:14:54,106 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/ad1a008a471c4807a5fda9e830abf337 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/ad1a008a471c4807a5fda9e830abf337 2024-11-21T21:14:54,111 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0357b2a5ad6dd1cf7bba0f542d57baae/C of 0357b2a5ad6dd1cf7bba0f542d57baae into ad1a008a471c4807a5fda9e830abf337(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:14:54,111 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0357b2a5ad6dd1cf7bba0f542d57baae: 2024-11-21T21:14:54,111 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., storeName=0357b2a5ad6dd1cf7bba0f542d57baae/C, priority=13, startTime=1732223693648; duration=0sec 2024-11-21T21:14:54,111 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:54,111 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0357b2a5ad6dd1cf7bba0f542d57baae:C 2024-11-21T21:14:54,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:54,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51840 deadline: 1732223754250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:54,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:54,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:54,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:54,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:54,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1732223754250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:54,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:54,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:54,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:54,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223754250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:54,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:54,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:54,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:54,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:54,322 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:54,327 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411215424bae0684947fe96253dcb6ab7bf49_0357b2a5ad6dd1cf7bba0f542d57baae to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215424bae0684947fe96253dcb6ab7bf49_0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:14:54,328 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/5ab2da46f1574bc492e8f81ae2e76f1f, store: [table=TestAcidGuarantees family=A region=0357b2a5ad6dd1cf7bba0f542d57baae] 2024-11-21T21:14:54,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/5ab2da46f1574bc492e8f81ae2e76f1f is 175, key is test_row_0/A:col10/1732223693900/Put/seqid=0 2024-11-21T21:14:54,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742004_1180 (size=39549) 2024-11-21T21:14:54,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T21:14:54,531 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:14:54,531 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:14:54,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-21T21:14:54,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T21:14:54,533 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:14:54,534 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:14:54,534 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:14:54,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:54,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51840 deadline: 1732223754560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:54,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:54,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:54,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:54,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1732223754560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:54,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:54,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:54,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223754560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:54,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:54,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:54,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:54,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:54,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:54,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:54,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T21:14:54,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T21:14:54,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:54,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. as already flushing 2024-11-21T21:14:54,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:54,688 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:54,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:54,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:54,733 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/5ab2da46f1574bc492e8f81ae2e76f1f 2024-11-21T21:14:54,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/5d0408c402574537931e3c7dd08decea is 50, key is test_row_0/B:col10/1732223693900/Put/seqid=0 2024-11-21T21:14:54,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742005_1181 (size=12001) 2024-11-21T21:14:54,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T21:14:54,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:54,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. as already flushing 2024-11-21T21:14:54,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:54,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:54,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:54,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:54,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T21:14:54,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:54,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51920 deadline: 1732223754930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:54,935 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,41213,1732223653469 for 'test_row_1' in 0357b2a5ad6dd1cf7bba0f542d57baae of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:14:54,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:54,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:54,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:54,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:54,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51874 deadline: 1732223754941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:54,942 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,41213,1732223653469 for 'test_row_1' in 0357b2a5ad6dd1cf7bba0f542d57baae of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:14:54,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:54,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:54,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:54,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T21:14:54,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:54,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. as already flushing 2024-11-21T21:14:54,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:54,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:54,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:54,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:55,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:55,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51840 deadline: 1732223755070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:55,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:55,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:55,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1732223755070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:55,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:55,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:55,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:55,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:55,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:55,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:55,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223755070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:55,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:55,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:55,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:55,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T21:14:55,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:55,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. as already flushing 2024-11-21T21:14:55,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:55,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:55,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:55,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:55,156 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/5d0408c402574537931e3c7dd08decea 2024-11-21T21:14:55,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T21:14:55,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/5656a04d399049ad9d36acf1c10d306b is 50, key is test_row_0/C:col10/1732223693900/Put/seqid=0 2024-11-21T21:14:55,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742006_1182 (size=12001) 2024-11-21T21:14:55,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T21:14:55,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:55,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. as already flushing 2024-11-21T21:14:55,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:55,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:55,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:55,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:55,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T21:14:55,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:55,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. as already flushing 2024-11-21T21:14:55,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:55,454 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:55,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:55,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:55,571 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/5656a04d399049ad9d36acf1c10d306b 2024-11-21T21:14:55,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/5ab2da46f1574bc492e8f81ae2e76f1f as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/5ab2da46f1574bc492e8f81ae2e76f1f 2024-11-21T21:14:55,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/5ab2da46f1574bc492e8f81ae2e76f1f, entries=200, sequenceid=120, filesize=38.6 K 2024-11-21T21:14:55,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/5d0408c402574537931e3c7dd08decea as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/5d0408c402574537931e3c7dd08decea 2024-11-21T21:14:55,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/5d0408c402574537931e3c7dd08decea, entries=150, sequenceid=120, filesize=11.7 K 2024-11-21T21:14:55,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/5656a04d399049ad9d36acf1c10d306b as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/5656a04d399049ad9d36acf1c10d306b 2024-11-21T21:14:55,596 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/5656a04d399049ad9d36acf1c10d306b, entries=150, sequenceid=120, filesize=11.7 K 2024-11-21T21:14:55,597 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 0357b2a5ad6dd1cf7bba0f542d57baae in 1695ms, sequenceid=120, compaction requested=false 2024-11-21T21:14:55,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0357b2a5ad6dd1cf7bba0f542d57baae: 2024-11-21T21:14:55,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T21:14:55,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:55,607 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 0357b2a5ad6dd1cf7bba0f542d57baae 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-21T21:14:55,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0357b2a5ad6dd1cf7bba0f542d57baae, store=A 2024-11-21T21:14:55,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:55,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0357b2a5ad6dd1cf7bba0f542d57baae, store=B 2024-11-21T21:14:55,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:55,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0357b2a5ad6dd1cf7bba0f542d57baae, store=C 2024-11-21T21:14:55,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:55,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411215eca576c22754fb8a037e3a0070cf1bb_0357b2a5ad6dd1cf7bba0f542d57baae is 50, key is test_row_0/A:col10/1732223693925/Put/seqid=0 2024-11-21T21:14:55,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742007_1183 (size=12254) 2024-11-21T21:14:55,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,652 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411215eca576c22754fb8a037e3a0070cf1bb_0357b2a5ad6dd1cf7bba0f542d57baae to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215eca576c22754fb8a037e3a0070cf1bb_0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:14:55,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/3718785a08aa4faca3bf1eafd379d6cf, store: [table=TestAcidGuarantees family=A region=0357b2a5ad6dd1cf7bba0f542d57baae] 2024-11-21T21:14:55,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/3718785a08aa4faca3bf1eafd379d6cf is 175, key is test_row_0/A:col10/1732223693925/Put/seqid=0 2024-11-21T21:14:55,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T21:14:55,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742008_1184 (size=31055) 2024-11-21T21:14:55,672 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=135, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/3718785a08aa4faca3bf1eafd379d6cf 2024-11-21T21:14:55,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/1464321b38a044878435b31a25d255a9 is 50, key is test_row_0/B:col10/1732223693925/Put/seqid=0 2024-11-21T21:14:55,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742009_1185 (size=12101) 2024-11-21T21:14:55,691 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/1464321b38a044878435b31a25d255a9 2024-11-21T21:14:55,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/aeecd218b3e84bfa9bee443c2bc1f6bc is 50, key is test_row_0/C:col10/1732223693925/Put/seqid=0 2024-11-21T21:14:55,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742010_1186 (size=12101) 2024-11-21T21:14:55,733 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/aeecd218b3e84bfa9bee443c2bc1f6bc 2024-11-21T21:14:55,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/3718785a08aa4faca3bf1eafd379d6cf as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/3718785a08aa4faca3bf1eafd379d6cf 2024-11-21T21:14:55,750 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/3718785a08aa4faca3bf1eafd379d6cf, entries=150, sequenceid=135, filesize=30.3 K 2024-11-21T21:14:55,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/1464321b38a044878435b31a25d255a9 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/1464321b38a044878435b31a25d255a9 2024-11-21T21:14:55,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,758 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/1464321b38a044878435b31a25d255a9, entries=150, sequenceid=135, filesize=11.8 K 2024-11-21T21:14:55,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/aeecd218b3e84bfa9bee443c2bc1f6bc as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/aeecd218b3e84bfa9bee443c2bc1f6bc 2024-11-21T21:14:55,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,769 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/aeecd218b3e84bfa9bee443c2bc1f6bc, entries=150, sequenceid=135, filesize=11.8 K 2024-11-21T21:14:55,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,770 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=0 B/0 for 0357b2a5ad6dd1cf7bba0f542d57baae in 163ms, sequenceid=135, compaction requested=true 2024-11-21T21:14:55,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 0357b2a5ad6dd1cf7bba0f542d57baae: 2024-11-21T21:14:55,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:55,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-21T21:14:55,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-21T21:14:55,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,775 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-21T21:14:55,775 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2380 sec 2024-11-21T21:14:55,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,777 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.2450 sec 2024-11-21T21:14:55,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:55,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:14:56,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,139 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0357b2a5ad6dd1cf7bba0f542d57baae 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T21:14:56,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0357b2a5ad6dd1cf7bba0f542d57baae, store=A 2024-11-21T21:14:56,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:56,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0357b2a5ad6dd1cf7bba0f542d57baae, store=B 2024-11-21T21:14:56,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:56,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0357b2a5ad6dd1cf7bba0f542d57baae, store=C 2024-11-21T21:14:56,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:56,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d62ba46cb2b149ceb139a7ef077f31ea_0357b2a5ad6dd1cf7bba0f542d57baae is 50, key is test_row_0/A:col10/1732223696125/Put/seqid=0 2024-11-21T21:14:56,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742011_1187 (size=22268) 2024-11-21T21:14:56,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,165 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,169 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d62ba46cb2b149ceb139a7ef077f31ea_0357b2a5ad6dd1cf7bba0f542d57baae to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d62ba46cb2b149ceb139a7ef077f31ea_0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:14:56,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,171 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/b196a54342b847409891cdd1ee8c7ce7, store: [table=TestAcidGuarantees family=A region=0357b2a5ad6dd1cf7bba0f542d57baae] 2024-11-21T21:14:56,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/b196a54342b847409891cdd1ee8c7ce7 is 175, key is test_row_0/A:col10/1732223696125/Put/seqid=0 2024-11-21T21:14:56,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:56,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742012_1188 (size=65673) 2024-11-21T21:14:56,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:56,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1732223756225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:56,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:56,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:56,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:56,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51840 deadline: 1732223756226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:56,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:56,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:56,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:56,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:56,228 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732223696205, max timestamp=1732223696224Num uniques 859; 2024-11-21T21:14:56,228 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T21:14:56,230 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732223696205, max timestamp=1732223696224Num uniques 859; 2024-11-21T21:14:56,230 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-21T21:14:56,231 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732223696205, max timestamp=1732223696224Num uniques 859; 2024-11-21T21:14:56,231 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-21T21:14:56,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:56,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223756319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:56,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:56,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:56,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:56,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:56,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1732223756330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:56,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:56,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:56,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:56,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:56,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51840 deadline: 1732223756330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:56,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:56,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:56,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:56,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:56,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223756430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:56,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:56,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:56,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:56,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:56,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1732223756540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:56,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:56,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:56,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:56,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:56,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51840 deadline: 1732223756540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:56,544 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:56,544 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:56,545 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:56,584 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=145, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/b196a54342b847409891cdd1ee8c7ce7 2024-11-21T21:14:56,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/e069054bcea44b9f9d86c59b92a4c880 is 50, key is test_row_0/B:col10/1732223696125/Put/seqid=0 2024-11-21T21:14:56,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742013_1189 (size=12151) 2024-11-21T21:14:56,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:56,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223756650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:56,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:56,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:56,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:56,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T21:14:56,681 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:14:56,681 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:14:56,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-21T21:14:56,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-21T21:14:56,683 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:14:56,684 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:14:56,684 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:14:56,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-21T21:14:56,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-21T21:14:56,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:56,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. as already flushing 2024-11-21T21:14:56,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:56,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:56,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:56,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:56,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:56,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1732223756850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:56,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:56,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:56,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:56,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:56,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51840 deadline: 1732223756850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:56,854 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:56,854 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:56,854 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:56,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:56,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51920 deadline: 1732223756950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:56,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:56,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51874 deadline: 1732223756950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:56,951 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,41213,1732223653469 for 'test_row_1' in 0357b2a5ad6dd1cf7bba0f542d57baae of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:14:56,952 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,41213,1732223653469 for 'test_row_1' in 0357b2a5ad6dd1cf7bba0f542d57baae of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:14:56,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:56,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:56,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:56,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:56,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:56,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:56,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:56,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223756960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:56,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:56,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:56,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:56,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-21T21:14:56,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:56,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. as already flushing 2024-11-21T21:14:56,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:56,989 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:56,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:56,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:56,999 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/e069054bcea44b9f9d86c59b92a4c880 2024-11-21T21:14:57,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-21T21:14:57,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/6ab43bcf58724f94bddbf1a71af364da is 50, key is test_row_0/C:col10/1732223696125/Put/seqid=0 2024-11-21T21:14:57,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742014_1190 (size=12151) 2024-11-21T21:14:57,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-21T21:14:57,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:57,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. as already flushing 2024-11-21T21:14:57,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:57,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:57,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:57,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-21T21:14:57,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:57,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. as already flushing 2024-11-21T21:14:57,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:57,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:57,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:57,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:14:57,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-21T21:14:57,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:57,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1732223757361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:57,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:57,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:57,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:57,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:57,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51840 deadline: 1732223757361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:57,366 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:57,366 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:57,366 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:57,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/6ab43bcf58724f94bddbf1a71af364da 2024-11-21T21:14:57,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/b196a54342b847409891cdd1ee8c7ce7 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/b196a54342b847409891cdd1ee8c7ce7 2024-11-21T21:14:57,420 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/b196a54342b847409891cdd1ee8c7ce7, entries=350, sequenceid=145, filesize=64.1 K 2024-11-21T21:14:57,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/e069054bcea44b9f9d86c59b92a4c880 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/e069054bcea44b9f9d86c59b92a4c880 2024-11-21T21:14:57,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/e069054bcea44b9f9d86c59b92a4c880, entries=150, sequenceid=145, filesize=11.9 K 2024-11-21T21:14:57,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/6ab43bcf58724f94bddbf1a71af364da as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/6ab43bcf58724f94bddbf1a71af364da 2024-11-21T21:14:57,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/6ab43bcf58724f94bddbf1a71af364da, entries=150, sequenceid=145, filesize=11.9 K 2024-11-21T21:14:57,431 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=135.63 KB/138880 for 0357b2a5ad6dd1cf7bba0f542d57baae in 1293ms, sequenceid=145, compaction requested=true 2024-11-21T21:14:57,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0357b2a5ad6dd1cf7bba0f542d57baae: 2024-11-21T21:14:57,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0357b2a5ad6dd1cf7bba0f542d57baae:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:14:57,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:57,432 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:14:57,432 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:14:57,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0357b2a5ad6dd1cf7bba0f542d57baae:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:14:57,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:57,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0357b2a5ad6dd1cf7bba0f542d57baae:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:14:57,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:14:57,434 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:14:57,434 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): 0357b2a5ad6dd1cf7bba0f542d57baae/B is initiating minor compaction (all files) 2024-11-21T21:14:57,434 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0357b2a5ad6dd1cf7bba0f542d57baae/B in TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:57,434 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 167438 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:14:57,434 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): 0357b2a5ad6dd1cf7bba0f542d57baae/A is initiating minor compaction (all files) 2024-11-21T21:14:57,434 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/cd9c6020491c4a73ad896307eaac6d43, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/5d0408c402574537931e3c7dd08decea, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/1464321b38a044878435b31a25d255a9, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/e069054bcea44b9f9d86c59b92a4c880] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp, totalSize=47.3 K 2024-11-21T21:14:57,434 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0357b2a5ad6dd1cf7bba0f542d57baae/A in TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:57,434 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/9da84e753020408788b2a0065a1d18e5, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/5ab2da46f1574bc492e8f81ae2e76f1f, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/3718785a08aa4faca3bf1eafd379d6cf, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/b196a54342b847409891cdd1ee8c7ce7] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp, totalSize=163.5 K 2024-11-21T21:14:57,434 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:57,434 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. files: [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/9da84e753020408788b2a0065a1d18e5, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/5ab2da46f1574bc492e8f81ae2e76f1f, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/3718785a08aa4faca3bf1eafd379d6cf, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/b196a54342b847409891cdd1ee8c7ce7] 2024-11-21T21:14:57,434 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting cd9c6020491c4a73ad896307eaac6d43, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732223692391 2024-11-21T21:14:57,435 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d0408c402574537931e3c7dd08decea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732223692745 2024-11-21T21:14:57,435 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9da84e753020408788b2a0065a1d18e5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732223692391 2024-11-21T21:14:57,435 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ab2da46f1574bc492e8f81ae2e76f1f, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732223692745 2024-11-21T21:14:57,435 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 1464321b38a044878435b31a25d255a9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732223693915 2024-11-21T21:14:57,435 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3718785a08aa4faca3bf1eafd379d6cf, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732223693915 2024-11-21T21:14:57,435 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting e069054bcea44b9f9d86c59b92a4c880, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732223696110 2024-11-21T21:14:57,435 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting b196a54342b847409891cdd1ee8c7ce7, keycount=350, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732223696110 2024-11-21T21:14:57,446 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0357b2a5ad6dd1cf7bba0f542d57baae#B#compaction#165 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:14:57,446 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/e280f96e13b144df92ec47b8c676a5bc is 50, key is test_row_0/B:col10/1732223696125/Put/seqid=0 2024-11-21T21:14:57,449 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0357b2a5ad6dd1cf7bba0f542d57baae] 2024-11-21T21:14:57,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-21T21:14:57,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:57,450 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 0357b2a5ad6dd1cf7bba0f542d57baae 3/3 column families, dataSize=135.63 KB heapSize=388.23 KB 2024-11-21T21:14:57,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0357b2a5ad6dd1cf7bba0f542d57baae, store=A 2024-11-21T21:14:57,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:14:57,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0357b2a5ad6dd1cf7bba0f542d57baae, store=B 2024-11-21T21:14:57,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:14:57,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0357b2a5ad6dd1cf7bba0f542d57baae, store=C 2024-11-21T21:14:57,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:14:57,457 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121618b1b4342fc49d0b9f88bf8a4f6d800_0357b2a5ad6dd1cf7bba0f542d57baae store=[table=TestAcidGuarantees family=A region=0357b2a5ad6dd1cf7bba0f542d57baae] 2024-11-21T21:14:57,459 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121618b1b4342fc49d0b9f88bf8a4f6d800_0357b2a5ad6dd1cf7bba0f542d57baae, store=[table=TestAcidGuarantees family=A region=0357b2a5ad6dd1cf7bba0f542d57baae] 2024-11-21T21:14:57,459 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121618b1b4342fc49d0b9f88bf8a4f6d800_0357b2a5ad6dd1cf7bba0f542d57baae because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0357b2a5ad6dd1cf7bba0f542d57baae] 2024-11-21T21:14:57,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742015_1191 (size=12493) 2024-11-21T21:14:57,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121fe84be96f983487c86692d4d29265d2e_0357b2a5ad6dd1cf7bba0f542d57baae is 50, key is test_row_0/A:col10/1732223696269/Put/seqid=0 2024-11-21T21:14:57,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742016_1192 (size=4469) 2024-11-21T21:14:57,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:14:57,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. as already flushing 2024-11-21T21:14:57,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742017_1193 (size=12304) 2024-11-21T21:14:57,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:57,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223757516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:57,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:57,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:57,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:57,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:57,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223757630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:57,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:57,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:57,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:57,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-21T21:14:57,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:57,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223757840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:57,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:57,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:57,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:57,875 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/e280f96e13b144df92ec47b8c676a5bc as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/e280f96e13b144df92ec47b8c676a5bc 2024-11-21T21:14:57,880 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0357b2a5ad6dd1cf7bba0f542d57baae/B of 0357b2a5ad6dd1cf7bba0f542d57baae into e280f96e13b144df92ec47b8c676a5bc(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:14:57,880 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0357b2a5ad6dd1cf7bba0f542d57baae: 2024-11-21T21:14:57,880 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., storeName=0357b2a5ad6dd1cf7bba0f542d57baae/B, priority=12, startTime=1732223697432; duration=0sec 2024-11-21T21:14:57,880 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:14:57,880 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0357b2a5ad6dd1cf7bba0f542d57baae:B 2024-11-21T21:14:57,880 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:14:57,881 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:14:57,882 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): 0357b2a5ad6dd1cf7bba0f542d57baae/C is initiating minor compaction (all files) 2024-11-21T21:14:57,882 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0357b2a5ad6dd1cf7bba0f542d57baae/C in TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:57,882 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/ad1a008a471c4807a5fda9e830abf337, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/5656a04d399049ad9d36acf1c10d306b, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/aeecd218b3e84bfa9bee443c2bc1f6bc, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/6ab43bcf58724f94bddbf1a71af364da] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp, totalSize=47.3 K 2024-11-21T21:14:57,882 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting ad1a008a471c4807a5fda9e830abf337, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732223692391 2024-11-21T21:14:57,882 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 5656a04d399049ad9d36acf1c10d306b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732223692745 2024-11-21T21:14:57,883 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting aeecd218b3e84bfa9bee443c2bc1f6bc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732223693915 2024-11-21T21:14:57,883 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ab43bcf58724f94bddbf1a71af364da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732223696110 2024-11-21T21:14:57,885 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0357b2a5ad6dd1cf7bba0f542d57baae#A#compaction#166 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:14:57,885 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/06c6d9c737f7465988009ed91fc05f41 is 175, key is test_row_0/A:col10/1732223696125/Put/seqid=0 2024-11-21T21:14:57,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742018_1194 (size=31447) 2024-11-21T21:14:57,894 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0357b2a5ad6dd1cf7bba0f542d57baae#C#compaction#168 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:14:57,894 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/d087ac4f72144d43bb1fb83979f274a2 is 50, key is test_row_0/C:col10/1732223696125/Put/seqid=0 2024-11-21T21:14:57,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:14:57,900 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121fe84be96f983487c86692d4d29265d2e_0357b2a5ad6dd1cf7bba0f542d57baae to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121fe84be96f983487c86692d4d29265d2e_0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:14:57,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/d7449eef41884cf9b6b8a0b483dd69a7, store: [table=TestAcidGuarantees family=A region=0357b2a5ad6dd1cf7bba0f542d57baae] 2024-11-21T21:14:57,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/d7449eef41884cf9b6b8a0b483dd69a7 is 175, key is test_row_0/A:col10/1732223696269/Put/seqid=0 2024-11-21T21:14:57,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742019_1195 (size=12493) 2024-11-21T21:14:57,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742020_1196 (size=31105) 2024-11-21T21:14:58,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:58,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223758150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:58,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:58,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:58,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:58,299 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/06c6d9c737f7465988009ed91fc05f41 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/06c6d9c737f7465988009ed91fc05f41 2024-11-21T21:14:58,306 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0357b2a5ad6dd1cf7bba0f542d57baae/A of 0357b2a5ad6dd1cf7bba0f542d57baae into 06c6d9c737f7465988009ed91fc05f41(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:14:58,306 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0357b2a5ad6dd1cf7bba0f542d57baae: 2024-11-21T21:14:58,306 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., storeName=0357b2a5ad6dd1cf7bba0f542d57baae/A, priority=12, startTime=1732223697432; duration=0sec 2024-11-21T21:14:58,306 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:58,306 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0357b2a5ad6dd1cf7bba0f542d57baae:A 2024-11-21T21:14:58,310 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=182, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/d7449eef41884cf9b6b8a0b483dd69a7 2024-11-21T21:14:58,318 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/d087ac4f72144d43bb1fb83979f274a2 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/d087ac4f72144d43bb1fb83979f274a2 2024-11-21T21:14:58,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/ea13f49e50c84e2abf5af4afbcc88552 is 50, key is test_row_0/B:col10/1732223696269/Put/seqid=0 2024-11-21T21:14:58,330 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0357b2a5ad6dd1cf7bba0f542d57baae/C of 0357b2a5ad6dd1cf7bba0f542d57baae into d087ac4f72144d43bb1fb83979f274a2(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:14:58,330 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0357b2a5ad6dd1cf7bba0f542d57baae: 2024-11-21T21:14:58,330 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., storeName=0357b2a5ad6dd1cf7bba0f542d57baae/C, priority=12, startTime=1732223697432; duration=0sec 2024-11-21T21:14:58,330 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:14:58,330 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0357b2a5ad6dd1cf7bba0f542d57baae:C 2024-11-21T21:14:58,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742021_1197 (size=12151) 2024-11-21T21:14:58,361 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/ea13f49e50c84e2abf5af4afbcc88552 2024-11-21T21:14:58,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:58,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1732223758370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:58,376 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,41213,1732223653469 for 'test_row_0' in 0357b2a5ad6dd1cf7bba0f542d57baae of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:14:58,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:58,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:58,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:58,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:58,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/1c2bb4d66e994bc9b886f2deff40a393 is 50, key is test_row_0/C:col10/1732223696269/Put/seqid=0 2024-11-21T21:14:58,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51840 deadline: 1732223758381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:58,384 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,41213,1732223653469 for 'test_row_1' in 0357b2a5ad6dd1cf7bba0f542d57baae of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:14:58,385 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:58,385 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:58,385 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:58,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742022_1198 (size=12151) 2024-11-21T21:14:58,612 INFO [master/849dee24a242:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-21T21:14:58,612 INFO [master/849dee24a242:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-21T21:14:58,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:58,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223758670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:58,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:58,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:58,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:58,812 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/1c2bb4d66e994bc9b886f2deff40a393 2024-11-21T21:14:58,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/d7449eef41884cf9b6b8a0b483dd69a7 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/d7449eef41884cf9b6b8a0b483dd69a7 2024-11-21T21:14:58,825 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/d7449eef41884cf9b6b8a0b483dd69a7, entries=150, sequenceid=182, filesize=30.4 K 2024-11-21T21:14:58,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/ea13f49e50c84e2abf5af4afbcc88552 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/ea13f49e50c84e2abf5af4afbcc88552 2024-11-21T21:14:58,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-21T21:14:58,841 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/ea13f49e50c84e2abf5af4afbcc88552, entries=150, sequenceid=182, filesize=11.9 K 2024-11-21T21:14:58,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/1c2bb4d66e994bc9b886f2deff40a393 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/1c2bb4d66e994bc9b886f2deff40a393 2024-11-21T21:14:58,855 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/1c2bb4d66e994bc9b886f2deff40a393, entries=150, sequenceid=182, filesize=11.9 K 2024-11-21T21:14:58,856 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~135.63 KB/138880, heapSize ~388.19 KB/397504, currentSize=46.96 KB/48090 for 0357b2a5ad6dd1cf7bba0f542d57baae in 1406ms, sequenceid=182, compaction requested=false 2024-11-21T21:14:58,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 0357b2a5ad6dd1cf7bba0f542d57baae: 2024-11-21T21:14:58,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:14:58,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-21T21:14:58,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-21T21:14:58,860 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-21T21:14:58,860 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1740 sec 2024-11-21T21:14:58,863 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.1810 sec 2024-11-21T21:14:59,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:14:59,682 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0357b2a5ad6dd1cf7bba0f542d57baae 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-21T21:14:59,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0357b2a5ad6dd1cf7bba0f542d57baae, store=A 2024-11-21T21:14:59,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:59,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0357b2a5ad6dd1cf7bba0f542d57baae, store=B 2024-11-21T21:14:59,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:59,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0357b2a5ad6dd1cf7bba0f542d57baae, store=C 2024-11-21T21:14:59,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:14:59,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411219a90a7ec10b4459881640c8874c62d6a_0357b2a5ad6dd1cf7bba0f542d57baae is 50, key is test_row_0/A:col10/1732223697514/Put/seqid=0 2024-11-21T21:14:59,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742023_1199 (size=14794) 2024-11-21T21:14:59,794 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732223699771, max timestamp=1732223699791Num uniques 859; 2024-11-21T21:14:59,794 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T21:14:59,796 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732223699771, max timestamp=1732223699791Num uniques 859; 2024-11-21T21:14:59,796 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732223699771, max timestamp=1732223699791Num uniques 859; 2024-11-21T21:14:59,796 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T21:14:59,796 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T21:14:59,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:59,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223759875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:59,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:59,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:59,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:14:59,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:14:59,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223759990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:14:59,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:59,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:14:59,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:00,118 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:15:00,122 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411219a90a7ec10b4459881640c8874c62d6a_0357b2a5ad6dd1cf7bba0f542d57baae to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219a90a7ec10b4459881640c8874c62d6a_0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:15:00,123 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/aacc5067551b4e5b9f35f6fb55a5a170, store: [table=TestAcidGuarantees family=A region=0357b2a5ad6dd1cf7bba0f542d57baae] 2024-11-21T21:15:00,123 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/aacc5067551b4e5b9f35f6fb55a5a170 is 175, key is test_row_0/A:col10/1732223697514/Put/seqid=0 2024-11-21T21:15:00,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742024_1200 (size=39749) 2024-11-21T21:15:00,127 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=196, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/aacc5067551b4e5b9f35f6fb55a5a170 2024-11-21T21:15:00,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/bb40aa3a0163476da6bfb16ca13ed9b6 is 50, key is test_row_0/B:col10/1732223697514/Put/seqid=0 2024-11-21T21:15:00,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742025_1201 (size=12151) 2024-11-21T21:15:00,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:00,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223760200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:00,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:00,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:00,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:00,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:00,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1732223760390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:00,391 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,41213,1732223653469 for 'test_row_0' in 0357b2a5ad6dd1cf7bba0f542d57baae of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:15:00,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:00,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:00,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:00,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:00,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51840 deadline: 1732223760400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:00,404 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,41213,1732223653469 for 'test_row_1' in 0357b2a5ad6dd1cf7bba0f542d57baae of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:15:00,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:00,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:00,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:00,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:00,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223760510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:00,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:00,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:00,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:00,538 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/bb40aa3a0163476da6bfb16ca13ed9b6 2024-11-21T21:15:00,545 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/7d340644b0304c60993ffb395d5eacb2 is 50, key is test_row_0/C:col10/1732223697514/Put/seqid=0 2024-11-21T21:15:00,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742026_1202 (size=12151) 2024-11-21T21:15:00,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-21T21:15:00,841 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:15:00,841 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:15:00,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-21T21:15:00,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T21:15:00,843 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:15:00,843 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:15:00,843 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:15:00,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T21:15:00,950 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/7d340644b0304c60993ffb395d5eacb2 2024-11-21T21:15:00,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/aacc5067551b4e5b9f35f6fb55a5a170 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/aacc5067551b4e5b9f35f6fb55a5a170 2024-11-21T21:15:00,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/aacc5067551b4e5b9f35f6fb55a5a170, entries=200, sequenceid=196, filesize=38.8 K 2024-11-21T21:15:00,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/bb40aa3a0163476da6bfb16ca13ed9b6 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/bb40aa3a0163476da6bfb16ca13ed9b6 2024-11-21T21:15:00,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:00,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51874 deadline: 1732223760961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:00,962 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,41213,1732223653469 for 'test_row_1' in 0357b2a5ad6dd1cf7bba0f542d57baae of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:15:00,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:00,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:00,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:00,963 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/bb40aa3a0163476da6bfb16ca13ed9b6, entries=150, sequenceid=196, filesize=11.9 K 2024-11-21T21:15:00,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/7d340644b0304c60993ffb395d5eacb2 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/7d340644b0304c60993ffb395d5eacb2 2024-11-21T21:15:00,968 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/7d340644b0304c60993ffb395d5eacb2, entries=150, sequenceid=196, filesize=11.9 K 2024-11-21T21:15:00,969 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 0357b2a5ad6dd1cf7bba0f542d57baae in 1287ms, sequenceid=196, compaction requested=true 2024-11-21T21:15:00,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0357b2a5ad6dd1cf7bba0f542d57baae: 2024-11-21T21:15:00,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0357b2a5ad6dd1cf7bba0f542d57baae:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:15:00,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:00,969 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:15:00,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0357b2a5ad6dd1cf7bba0f542d57baae:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:15:00,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:00,969 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:15:00,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0357b2a5ad6dd1cf7bba0f542d57baae:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:15:00,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:15:00,973 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:15:00,973 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:15:00,973 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): 0357b2a5ad6dd1cf7bba0f542d57baae/B is initiating minor compaction (all files) 2024-11-21T21:15:00,973 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): 0357b2a5ad6dd1cf7bba0f542d57baae/A is initiating minor compaction (all files) 2024-11-21T21:15:00,973 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0357b2a5ad6dd1cf7bba0f542d57baae/B in TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:15:00,973 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0357b2a5ad6dd1cf7bba0f542d57baae/A in TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:15:00,973 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/06c6d9c737f7465988009ed91fc05f41, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/d7449eef41884cf9b6b8a0b483dd69a7, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/aacc5067551b4e5b9f35f6fb55a5a170] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp, totalSize=99.9 K 2024-11-21T21:15:00,973 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/e280f96e13b144df92ec47b8c676a5bc, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/ea13f49e50c84e2abf5af4afbcc88552, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/bb40aa3a0163476da6bfb16ca13ed9b6] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp, totalSize=35.9 K 2024-11-21T21:15:00,973 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:15:00,973 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. files: [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/06c6d9c737f7465988009ed91fc05f41, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/d7449eef41884cf9b6b8a0b483dd69a7, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/aacc5067551b4e5b9f35f6fb55a5a170] 2024-11-21T21:15:00,974 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06c6d9c737f7465988009ed91fc05f41, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732223696110 2024-11-21T21:15:00,974 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting e280f96e13b144df92ec47b8c676a5bc, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732223696110 2024-11-21T21:15:00,974 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting ea13f49e50c84e2abf5af4afbcc88552, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732223696269 2024-11-21T21:15:00,974 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7449eef41884cf9b6b8a0b483dd69a7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732223696269 2024-11-21T21:15:00,974 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting aacc5067551b4e5b9f35f6fb55a5a170, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732223697482 2024-11-21T21:15:00,974 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting bb40aa3a0163476da6bfb16ca13ed9b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732223697503 2024-11-21T21:15:00,992 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0357b2a5ad6dd1cf7bba0f542d57baae] 2024-11-21T21:15:00,993 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0357b2a5ad6dd1cf7bba0f542d57baae#B#compaction#174 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:00,994 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/a897c6ac9d5b4f1884ffda8c866b17b1 is 50, key is test_row_0/B:col10/1732223697514/Put/seqid=0 2024-11-21T21:15:00,994 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121f268fd8d75c943b0be7f4bdb8ff276ec_0357b2a5ad6dd1cf7bba0f542d57baae store=[table=TestAcidGuarantees family=A region=0357b2a5ad6dd1cf7bba0f542d57baae] 2024-11-21T21:15:00,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-21T21:15:00,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:15:00,995 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 0357b2a5ad6dd1cf7bba0f542d57baae 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-21T21:15:00,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0357b2a5ad6dd1cf7bba0f542d57baae, store=A 2024-11-21T21:15:00,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:15:00,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0357b2a5ad6dd1cf7bba0f542d57baae, store=B 2024-11-21T21:15:00,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:15:00,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0357b2a5ad6dd1cf7bba0f542d57baae, store=C 2024-11-21T21:15:00,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:15:00,996 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121f268fd8d75c943b0be7f4bdb8ff276ec_0357b2a5ad6dd1cf7bba0f542d57baae, store=[table=TestAcidGuarantees family=A region=0357b2a5ad6dd1cf7bba0f542d57baae] 2024-11-21T21:15:00,996 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121f268fd8d75c943b0be7f4bdb8ff276ec_0357b2a5ad6dd1cf7bba0f542d57baae because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0357b2a5ad6dd1cf7bba0f542d57baae] 2024-11-21T21:15:01,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:15:01,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. as already flushing 2024-11-21T21:15:01,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742027_1203 (size=12595) 2024-11-21T21:15:01,024 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/a897c6ac9d5b4f1884ffda8c866b17b1 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/a897c6ac9d5b4f1884ffda8c866b17b1 2024-11-21T21:15:01,029 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0357b2a5ad6dd1cf7bba0f542d57baae/B of 0357b2a5ad6dd1cf7bba0f542d57baae into a897c6ac9d5b4f1884ffda8c866b17b1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:01,030 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0357b2a5ad6dd1cf7bba0f542d57baae: 2024-11-21T21:15:01,030 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., storeName=0357b2a5ad6dd1cf7bba0f542d57baae/B, priority=13, startTime=1732223700969; duration=0sec 2024-11-21T21:15:01,030 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:15:01,030 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0357b2a5ad6dd1cf7bba0f542d57baae:B 2024-11-21T21:15:01,030 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:15:01,031 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:15:01,031 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): 0357b2a5ad6dd1cf7bba0f542d57baae/C is initiating minor compaction (all files) 2024-11-21T21:15:01,031 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0357b2a5ad6dd1cf7bba0f542d57baae/C in TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:15:01,031 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/d087ac4f72144d43bb1fb83979f274a2, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/1c2bb4d66e994bc9b886f2deff40a393, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/7d340644b0304c60993ffb395d5eacb2] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp, totalSize=35.9 K 2024-11-21T21:15:01,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121990d8d3c710a4d84a100e8b2a8f1eba5_0357b2a5ad6dd1cf7bba0f542d57baae is 50, key is test_row_0/A:col10/1732223699853/Put/seqid=0 2024-11-21T21:15:01,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742028_1204 (size=4469) 2024-11-21T21:15:01,033 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting d087ac4f72144d43bb1fb83979f274a2, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732223696110 2024-11-21T21:15:01,034 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c2bb4d66e994bc9b886f2deff40a393, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732223696269 2024-11-21T21:15:01,034 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0357b2a5ad6dd1cf7bba0f542d57baae#A#compaction#175 average throughput is 0.58 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:01,034 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d340644b0304c60993ffb395d5eacb2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732223697503 2024-11-21T21:15:01,034 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/100a6869126c4f388de925f23c1b8a51 is 175, key is test_row_0/A:col10/1732223697514/Put/seqid=0 2024-11-21T21:15:01,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:01,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:01,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51920 deadline: 1732223761041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:01,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223761042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:01,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:01,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:01,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:01,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:01,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:01,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:01,046 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0357b2a5ad6dd1cf7bba0f542d57baae#C#compaction#177 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:01,046 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/9cd1469b12684514a07d5e9a7ef4b49d is 50, key is test_row_0/C:col10/1732223697514/Put/seqid=0 2024-11-21T21:15:01,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742030_1206 (size=31549) 2024-11-21T21:15:01,054 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/100a6869126c4f388de925f23c1b8a51 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/100a6869126c4f388de925f23c1b8a51 2024-11-21T21:15:01,062 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0357b2a5ad6dd1cf7bba0f542d57baae/A of 0357b2a5ad6dd1cf7bba0f542d57baae into 100a6869126c4f388de925f23c1b8a51(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:01,062 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0357b2a5ad6dd1cf7bba0f542d57baae: 2024-11-21T21:15:01,062 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., storeName=0357b2a5ad6dd1cf7bba0f542d57baae/A, priority=13, startTime=1732223700969; duration=0sec 2024-11-21T21:15:01,062 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:01,062 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0357b2a5ad6dd1cf7bba0f542d57baae:A 2024-11-21T21:15:01,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742029_1205 (size=12304) 2024-11-21T21:15:01,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:15:01,070 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121990d8d3c710a4d84a100e8b2a8f1eba5_0357b2a5ad6dd1cf7bba0f542d57baae to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121990d8d3c710a4d84a100e8b2a8f1eba5_0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:15:01,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/37ac9d2ab5744a9080ad9b6a662bd215, store: [table=TestAcidGuarantees family=A region=0357b2a5ad6dd1cf7bba0f542d57baae] 2024-11-21T21:15:01,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/37ac9d2ab5744a9080ad9b6a662bd215 is 175, key is test_row_0/A:col10/1732223699853/Put/seqid=0 2024-11-21T21:15:01,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742031_1207 (size=12595) 2024-11-21T21:15:01,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742032_1208 (size=31105) 2024-11-21T21:15:01,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:01,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223761151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:01,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:01,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:01,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51920 deadline: 1732223761151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:01,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:01,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:01,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:01,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:01,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:01,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T21:15:01,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:01,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223761360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:01,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:01,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51920 deadline: 1732223761360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:01,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:01,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:01,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:01,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:01,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:01,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:01,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T21:15:01,487 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/9cd1469b12684514a07d5e9a7ef4b49d as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/9cd1469b12684514a07d5e9a7ef4b49d 2024-11-21T21:15:01,489 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=231, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/37ac9d2ab5744a9080ad9b6a662bd215 2024-11-21T21:15:01,491 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0357b2a5ad6dd1cf7bba0f542d57baae/C of 0357b2a5ad6dd1cf7bba0f542d57baae into 9cd1469b12684514a07d5e9a7ef4b49d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:01,491 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0357b2a5ad6dd1cf7bba0f542d57baae: 2024-11-21T21:15:01,491 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., storeName=0357b2a5ad6dd1cf7bba0f542d57baae/C, priority=13, startTime=1732223700969; duration=0sec 2024-11-21T21:15:01,491 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:01,491 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0357b2a5ad6dd1cf7bba0f542d57baae:C 2024-11-21T21:15:01,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/d686f35903d0455fbf41aba322a8b897 is 50, key is test_row_0/B:col10/1732223699853/Put/seqid=0 2024-11-21T21:15:01,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742033_1209 (size=12151) 2024-11-21T21:15:01,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:01,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223761670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:01,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:01,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:01,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:01,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:01,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51920 deadline: 1732223761670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:01,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:01,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:01,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:01,901 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/d686f35903d0455fbf41aba322a8b897 2024-11-21T21:15:01,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/d38fabcec09d4b99a4debc3e5000e275 is 50, key is test_row_0/C:col10/1732223699853/Put/seqid=0 2024-11-21T21:15:01,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742034_1210 (size=12151) 2024-11-21T21:15:01,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T21:15:02,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:02,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223762180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:02,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:02,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:02,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:02,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51920 deadline: 1732223762180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:02,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:02,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:02,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:02,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:02,314 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/d38fabcec09d4b99a4debc3e5000e275 2024-11-21T21:15:02,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/37ac9d2ab5744a9080ad9b6a662bd215 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/37ac9d2ab5744a9080ad9b6a662bd215 2024-11-21T21:15:02,323 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/37ac9d2ab5744a9080ad9b6a662bd215, entries=150, sequenceid=231, filesize=30.4 K 2024-11-21T21:15:02,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/d686f35903d0455fbf41aba322a8b897 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/d686f35903d0455fbf41aba322a8b897 2024-11-21T21:15:02,328 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/d686f35903d0455fbf41aba322a8b897, entries=150, sequenceid=231, filesize=11.9 K 2024-11-21T21:15:02,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/d38fabcec09d4b99a4debc3e5000e275 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/d38fabcec09d4b99a4debc3e5000e275 2024-11-21T21:15:02,333 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/d38fabcec09d4b99a4debc3e5000e275, entries=150, sequenceid=231, filesize=11.9 K 2024-11-21T21:15:02,334 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=60.38 KB/61830 for 0357b2a5ad6dd1cf7bba0f542d57baae in 1339ms, sequenceid=231, compaction requested=false 2024-11-21T21:15:02,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 0357b2a5ad6dd1cf7bba0f542d57baae: 2024-11-21T21:15:02,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:15:02,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-21T21:15:02,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-21T21:15:02,336 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-21T21:15:02,336 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4920 sec 2024-11-21T21:15:02,338 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.4960 sec 2024-11-21T21:15:02,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T21:15:02,991 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:15:02,991 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:15:02,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-21T21:15:02,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T21:15:02,993 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:15:02,993 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:15:02,994 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:15:03,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T21:15:03,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-21T21:15:03,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:15:03,146 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 0357b2a5ad6dd1cf7bba0f542d57baae 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-21T21:15:03,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0357b2a5ad6dd1cf7bba0f542d57baae, store=A 2024-11-21T21:15:03,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:03,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0357b2a5ad6dd1cf7bba0f542d57baae, store=B 2024-11-21T21:15:03,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:03,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0357b2a5ad6dd1cf7bba0f542d57baae, store=C 2024-11-21T21:15:03,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:03,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112186ccfc30eca14c6db0c99cd40c6de61c_0357b2a5ad6dd1cf7bba0f542d57baae is 50, key is test_row_0/A:col10/1732223701002/Put/seqid=0 2024-11-21T21:15:03,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742035_1211 (size=12304) 2024-11-21T21:15:03,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:15:03,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. as already flushing 2024-11-21T21:15:03,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732223703228, max timestamp=1732223703240Num uniques 859; 2024-11-21T21:15:03,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T21:15:03,246 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732223703228, max timestamp=1732223703240Num uniques 859; 2024-11-21T21:15:03,246 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732223703228, max timestamp=1732223703240Num uniques 859; 2024-11-21T21:15:03,246 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T21:15:03,246 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T21:15:03,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:03,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223763274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:03,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:03,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:03,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:03,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:03,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51920 deadline: 1732223763276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:03,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:03,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:03,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:03,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T21:15:03,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:03,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223763380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:03,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:03,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:03,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:03,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:03,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51920 deadline: 1732223763390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:03,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:03,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:03,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:03,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:15:03,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:03,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223763591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:03,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:03,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:03,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:03,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:03,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51920 deadline: 1732223763600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:03,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:03,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:03,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:03,615 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112186ccfc30eca14c6db0c99cd40c6de61c_0357b2a5ad6dd1cf7bba0f542d57baae to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112186ccfc30eca14c6db0c99cd40c6de61c_0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:15:03,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/4be429ae72a240aa8d2439591e90dee4, store: [table=TestAcidGuarantees family=A region=0357b2a5ad6dd1cf7bba0f542d57baae] 2024-11-21T21:15:03,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/4be429ae72a240aa8d2439591e90dee4 is 175, key is test_row_0/A:col10/1732223701002/Put/seqid=0 2024-11-21T21:15:03,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T21:15:03,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742036_1212 (size=31105) 2024-11-21T21:15:03,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:03,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223763900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:03,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:03,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:03,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:03,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:03,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51920 deadline: 1732223763910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:03,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:03,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:03,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:04,022 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=246, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/4be429ae72a240aa8d2439591e90dee4 2024-11-21T21:15:04,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/d2f95036e9d645118d7cd355670e5518 is 50, key is test_row_0/B:col10/1732223701002/Put/seqid=0 2024-11-21T21:15:04,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742037_1213 (size=12151) 2024-11-21T21:15:04,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T21:15:04,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:04,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51896 deadline: 1732223764410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:04,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:04,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223764410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:04,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:04,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:04,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:04,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51840 deadline: 1732223764410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:04,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:04,418 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,41213,1732223653469 for 'test_row_0' in 0357b2a5ad6dd1cf7bba0f542d57baae of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:15:04,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:04,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:04,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:04,418 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,41213,1732223653469 for 'test_row_1' in 0357b2a5ad6dd1cf7bba0f542d57baae of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:15:04,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:04,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:04,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:04,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:04,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51920 deadline: 1732223764420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:04,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:04,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:04,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:04,434 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/d2f95036e9d645118d7cd355670e5518 2024-11-21T21:15:04,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/b7fce2561d034d03907788455623c07c is 50, key is test_row_0/C:col10/1732223701002/Put/seqid=0 2024-11-21T21:15:04,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742038_1214 (size=12151) 2024-11-21T21:15:04,849 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/b7fce2561d034d03907788455623c07c 2024-11-21T21:15:04,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/4be429ae72a240aa8d2439591e90dee4 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/4be429ae72a240aa8d2439591e90dee4 2024-11-21T21:15:04,859 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/4be429ae72a240aa8d2439591e90dee4, entries=150, sequenceid=246, filesize=30.4 K 2024-11-21T21:15:04,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/d2f95036e9d645118d7cd355670e5518 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/d2f95036e9d645118d7cd355670e5518 2024-11-21T21:15:04,864 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/d2f95036e9d645118d7cd355670e5518, entries=150, sequenceid=246, filesize=11.9 K 2024-11-21T21:15:04,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/b7fce2561d034d03907788455623c07c as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/b7fce2561d034d03907788455623c07c 2024-11-21T21:15:04,871 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/b7fce2561d034d03907788455623c07c, entries=150, sequenceid=246, filesize=11.9 K 2024-11-21T21:15:04,872 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 0357b2a5ad6dd1cf7bba0f542d57baae in 1726ms, sequenceid=246, compaction requested=true 2024-11-21T21:15:04,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 0357b2a5ad6dd1cf7bba0f542d57baae: 2024-11-21T21:15:04,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:15:04,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-21T21:15:04,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-21T21:15:04,875 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-21T21:15:04,875 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8810 sec 2024-11-21T21:15:04,877 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.8850 sec 2024-11-21T21:15:05,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T21:15:05,140 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:15:05,140 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:15:05,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-21T21:15:05,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-21T21:15:05,142 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:15:05,143 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:15:05,143 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:15:05,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-21T21:15:05,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-21T21:15:05,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:15:05,295 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 0357b2a5ad6dd1cf7bba0f542d57baae 3/3 column families, dataSize=120.76 KB heapSize=340.45 KB 2024-11-21T21:15:05,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0357b2a5ad6dd1cf7bba0f542d57baae, store=A 2024-11-21T21:15:05,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:15:05,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0357b2a5ad6dd1cf7bba0f542d57baae, store=B 2024-11-21T21:15:05,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:15:05,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0357b2a5ad6dd1cf7bba0f542d57baae, store=C 2024-11-21T21:15:05,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:15:05,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121226e63ae98e346d993694c711faa5ce4_0357b2a5ad6dd1cf7bba0f542d57baae is 50, key is test_row_0/A:col10/1732223703269/Put/seqid=0 2024-11-21T21:15:05,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742039_1215 (size=12454) 2024-11-21T21:15:05,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:15:05,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. as already flushing 2024-11-21T21:15:05,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:05,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51920 deadline: 1732223765448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:05,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:05,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:05,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:05,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:05,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223765449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:05,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:05,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:05,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:05,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-21T21:15:05,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:05,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51920 deadline: 1732223765560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:05,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:05,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:05,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223765560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:05,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:05,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:05,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:05,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:05,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:05,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:15:05,712 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121226e63ae98e346d993694c711faa5ce4_0357b2a5ad6dd1cf7bba0f542d57baae to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121226e63ae98e346d993694c711faa5ce4_0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:15:05,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/58d6b5dac12342f0926e190d2aaa87c6, store: [table=TestAcidGuarantees family=A region=0357b2a5ad6dd1cf7bba0f542d57baae] 2024-11-21T21:15:05,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/58d6b5dac12342f0926e190d2aaa87c6 is 175, key is test_row_0/A:col10/1732223703269/Put/seqid=0 2024-11-21T21:15:05,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742040_1216 (size=31255) 2024-11-21T21:15:05,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-21T21:15:05,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:05,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51920 deadline: 1732223765770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:05,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:05,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223765770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:05,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:05,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:05,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:05,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:05,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:05,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:06,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:06,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51920 deadline: 1732223766080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:06,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:06,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223766080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:06,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:06,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:06,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:06,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:06,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:06,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:06,118 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=280, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/58d6b5dac12342f0926e190d2aaa87c6 2024-11-21T21:15:06,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/874795ad81e14853aae67b05614eef19 is 50, key is test_row_0/B:col10/1732223703269/Put/seqid=0 2024-11-21T21:15:06,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742041_1217 (size=12301) 2024-11-21T21:15:06,130 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/874795ad81e14853aae67b05614eef19 2024-11-21T21:15:06,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/b188e5a622a94e9eb55d296655b58b50 is 50, key is test_row_0/C:col10/1732223703269/Put/seqid=0 2024-11-21T21:15:06,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742042_1218 (size=12301) 2024-11-21T21:15:06,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-21T21:15:06,543 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/b188e5a622a94e9eb55d296655b58b50 2024-11-21T21:15:06,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/58d6b5dac12342f0926e190d2aaa87c6 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/58d6b5dac12342f0926e190d2aaa87c6 2024-11-21T21:15:06,553 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/58d6b5dac12342f0926e190d2aaa87c6, entries=150, sequenceid=280, filesize=30.5 K 2024-11-21T21:15:06,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/874795ad81e14853aae67b05614eef19 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/874795ad81e14853aae67b05614eef19 2024-11-21T21:15:06,558 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/874795ad81e14853aae67b05614eef19, entries=150, sequenceid=280, filesize=12.0 K 2024-11-21T21:15:06,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/b188e5a622a94e9eb55d296655b58b50 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/b188e5a622a94e9eb55d296655b58b50 2024-11-21T21:15:06,567 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/b188e5a622a94e9eb55d296655b58b50, entries=150, sequenceid=280, filesize=12.0 K 2024-11-21T21:15:06,568 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~340.41 KB/348576, currentSize=60.38 KB/61830 for 0357b2a5ad6dd1cf7bba0f542d57baae in 1273ms, sequenceid=280, compaction requested=true 2024-11-21T21:15:06,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 0357b2a5ad6dd1cf7bba0f542d57baae: 2024-11-21T21:15:06,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:15:06,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-21T21:15:06,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-21T21:15:06,571 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-21T21:15:06,571 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4260 sec 2024-11-21T21:15:06,572 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.4310 sec 2024-11-21T21:15:06,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:15:06,592 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0357b2a5ad6dd1cf7bba0f542d57baae 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-21T21:15:06,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0357b2a5ad6dd1cf7bba0f542d57baae, store=A 2024-11-21T21:15:06,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:06,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0357b2a5ad6dd1cf7bba0f542d57baae, store=B 2024-11-21T21:15:06,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:06,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0357b2a5ad6dd1cf7bba0f542d57baae, store=C 2024-11-21T21:15:06,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:06,599 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121498e6787b1b448a296e7322585269e03_0357b2a5ad6dd1cf7bba0f542d57baae is 50, key is test_row_0/A:col10/1732223706591/Put/seqid=0 2024-11-21T21:15:06,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742043_1219 (size=12454) 2024-11-21T21:15:06,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:06,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223766632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:06,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:06,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:06,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:06,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:06,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51920 deadline: 1732223766640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:06,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:06,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:06,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:06,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:06,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223766750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:06,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:06,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:06,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51920 deadline: 1732223766750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:06,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:06,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:06,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:06,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:06,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:06,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:06,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223766960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:06,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:06,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51920 deadline: 1732223766961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:06,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:06,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:06,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:06,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:06,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:06,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:07,003 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:15:07,013 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121498e6787b1b448a296e7322585269e03_0357b2a5ad6dd1cf7bba0f542d57baae to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121498e6787b1b448a296e7322585269e03_0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:15:07,014 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/ef8ad262ab9e4560908bfe47fd97017c, store: [table=TestAcidGuarantees family=A region=0357b2a5ad6dd1cf7bba0f542d57baae] 2024-11-21T21:15:07,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/ef8ad262ab9e4560908bfe47fd97017c is 175, key is test_row_0/A:col10/1732223706591/Put/seqid=0 2024-11-21T21:15:07,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742044_1220 (size=31255) 2024-11-21T21:15:07,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:07,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:07,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223767270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:07,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51920 deadline: 1732223767270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:07,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:07,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:07,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:07,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:07,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:07,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:07,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-21T21:15:07,291 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:15:07,291 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:15:07,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-21T21:15:07,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T21:15:07,293 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:15:07,294 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:15:07,294 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:15:07,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T21:15:07,419 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=293, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/ef8ad262ab9e4560908bfe47fd97017c 2024-11-21T21:15:07,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/a5c1e10c974a41e083f6ec7f2abcdec2 is 50, key is test_row_0/B:col10/1732223706591/Put/seqid=0 2024-11-21T21:15:07,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742045_1221 (size=12301) 2024-11-21T21:15:07,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-21T21:15:07,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:15:07,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. as already flushing 2024-11-21T21:15:07,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:15:07,446 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:07,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:07,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:07,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-21T21:15:07,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:15:07,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. as already flushing 2024-11-21T21:15:07,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:15:07,599 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:07,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:07,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:07,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T21:15:07,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-21T21:15:07,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:15:07,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. as already flushing 2024-11-21T21:15:07,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:15:07,752 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:07,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:07,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:07,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:07,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51890 deadline: 1732223767780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:07,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:07,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:07,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:07,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:07,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51920 deadline: 1732223767790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:07,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:07,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:07,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:07,829 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/a5c1e10c974a41e083f6ec7f2abcdec2 2024-11-21T21:15:07,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/0e98013a03ff4cbc94453c2a5006e4be is 50, key is test_row_0/C:col10/1732223706591/Put/seqid=0 2024-11-21T21:15:07,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742046_1222 (size=12301) 2024-11-21T21:15:07,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-21T21:15:07,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:15:07,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. as already flushing 2024-11-21T21:15:07,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:15:07,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:07,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:07,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:07,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T21:15:08,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-21T21:15:08,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:15:08,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. as already flushing 2024-11-21T21:15:08,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:15:08,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:08,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:08,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:08,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-21T21:15:08,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:15:08,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. as already flushing 2024-11-21T21:15:08,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:15:08,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:08,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:08,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:08,211 INFO [Thread-743 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-743. 2024-11-21T21:15:08,211 DEBUG [Thread-743 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:15:08,211 INFO [Thread-747 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-747. 2024-11-21T21:15:08,212 DEBUG [Thread-743 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:08,212 DEBUG [Thread-747 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:15:08,212 DEBUG [Thread-747 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:08,212 DEBUG [Thread-743 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:08,212 DEBUG [Thread-747 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:08,212 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:15:08,212 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:15:08,215 INFO [Thread-745 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-745. 2024-11-21T21:15:08,215 DEBUG [Thread-745 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:15:08,215 DEBUG [Thread-745 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:08,215 DEBUG [Thread-745 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:08,215 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:15:08,218 INFO [Thread-749 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-749. 2024-11-21T21:15:08,218 DEBUG [Thread-749 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:15:08,218 DEBUG [Thread-749 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:08,218 DEBUG [Thread-749 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:08,218 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:15:08,241 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/0e98013a03ff4cbc94453c2a5006e4be 2024-11-21T21:15:08,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/ef8ad262ab9e4560908bfe47fd97017c as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/ef8ad262ab9e4560908bfe47fd97017c 2024-11-21T21:15:08,253 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/ef8ad262ab9e4560908bfe47fd97017c, entries=150, sequenceid=293, filesize=30.5 K 2024-11-21T21:15:08,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/a5c1e10c974a41e083f6ec7f2abcdec2 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/a5c1e10c974a41e083f6ec7f2abcdec2 2024-11-21T21:15:08,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/a5c1e10c974a41e083f6ec7f2abcdec2, entries=150, sequenceid=293, filesize=12.0 K 2024-11-21T21:15:08,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/0e98013a03ff4cbc94453c2a5006e4be as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/0e98013a03ff4cbc94453c2a5006e4be 2024-11-21T21:15:08,261 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/0e98013a03ff4cbc94453c2a5006e4be, entries=150, sequenceid=293, filesize=12.0 K 2024-11-21T21:15:08,262 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 0357b2a5ad6dd1cf7bba0f542d57baae in 1670ms, sequenceid=293, compaction requested=true 2024-11-21T21:15:08,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0357b2a5ad6dd1cf7bba0f542d57baae: 2024-11-21T21:15:08,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0357b2a5ad6dd1cf7bba0f542d57baae:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:15:08,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:08,262 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-21T21:15:08,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0357b2a5ad6dd1cf7bba0f542d57baae:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:15:08,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:08,262 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-21T21:15:08,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0357b2a5ad6dd1cf7bba0f542d57baae:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:15:08,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:15:08,264 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61499 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-21T21:15:08,264 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 156269 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-21T21:15:08,264 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): 0357b2a5ad6dd1cf7bba0f542d57baae/B is initiating minor compaction (all files) 2024-11-21T21:15:08,264 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): 0357b2a5ad6dd1cf7bba0f542d57baae/A is initiating minor compaction (all files) 2024-11-21T21:15:08,264 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0357b2a5ad6dd1cf7bba0f542d57baae/B in TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:15:08,264 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0357b2a5ad6dd1cf7bba0f542d57baae/A in TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:15:08,264 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/a897c6ac9d5b4f1884ffda8c866b17b1, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/d686f35903d0455fbf41aba322a8b897, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/d2f95036e9d645118d7cd355670e5518, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/874795ad81e14853aae67b05614eef19, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/a5c1e10c974a41e083f6ec7f2abcdec2] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp, totalSize=60.1 K 2024-11-21T21:15:08,264 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/100a6869126c4f388de925f23c1b8a51, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/37ac9d2ab5744a9080ad9b6a662bd215, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/4be429ae72a240aa8d2439591e90dee4, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/58d6b5dac12342f0926e190d2aaa87c6, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/ef8ad262ab9e4560908bfe47fd97017c] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp, totalSize=152.6 K 2024-11-21T21:15:08,264 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:15:08,264 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. files: [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/100a6869126c4f388de925f23c1b8a51, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/37ac9d2ab5744a9080ad9b6a662bd215, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/4be429ae72a240aa8d2439591e90dee4, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/58d6b5dac12342f0926e190d2aaa87c6, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/ef8ad262ab9e4560908bfe47fd97017c] 2024-11-21T21:15:08,264 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting a897c6ac9d5b4f1884ffda8c866b17b1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732223697503 2024-11-21T21:15:08,264 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 100a6869126c4f388de925f23c1b8a51, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732223697503 2024-11-21T21:15:08,265 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting d686f35903d0455fbf41aba322a8b897, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732223699845 2024-11-21T21:15:08,265 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 37ac9d2ab5744a9080ad9b6a662bd215, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732223699845 2024-11-21T21:15:08,265 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2f95036e9d645118d7cd355670e5518, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732223701002 2024-11-21T21:15:08,265 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 4be429ae72a240aa8d2439591e90dee4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732223701002 2024-11-21T21:15:08,265 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 874795ad81e14853aae67b05614eef19, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732223703269 2024-11-21T21:15:08,265 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 58d6b5dac12342f0926e190d2aaa87c6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732223703269 2024-11-21T21:15:08,265 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5c1e10c974a41e083f6ec7f2abcdec2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732223705434 2024-11-21T21:15:08,266 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting ef8ad262ab9e4560908bfe47fd97017c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732223705434 2024-11-21T21:15:08,277 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0357b2a5ad6dd1cf7bba0f542d57baae] 2024-11-21T21:15:08,278 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0357b2a5ad6dd1cf7bba0f542d57baae#B#compaction#189 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:08,278 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/bf24f162516a4a628ba762d68b6fef91 is 50, key is test_row_0/B:col10/1732223706591/Put/seqid=0 2024-11-21T21:15:08,279 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121ac201d173dba40dfbd8db0d9b7cc0215_0357b2a5ad6dd1cf7bba0f542d57baae store=[table=TestAcidGuarantees family=A region=0357b2a5ad6dd1cf7bba0f542d57baae] 2024-11-21T21:15:08,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742047_1223 (size=12915) 2024-11-21T21:15:08,293 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/bf24f162516a4a628ba762d68b6fef91 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/bf24f162516a4a628ba762d68b6fef91 2024-11-21T21:15:08,300 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 0357b2a5ad6dd1cf7bba0f542d57baae/B of 0357b2a5ad6dd1cf7bba0f542d57baae into bf24f162516a4a628ba762d68b6fef91(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:08,300 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0357b2a5ad6dd1cf7bba0f542d57baae: 2024-11-21T21:15:08,300 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., storeName=0357b2a5ad6dd1cf7bba0f542d57baae/B, priority=11, startTime=1732223708262; duration=0sec 2024-11-21T21:15:08,300 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:15:08,300 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0357b2a5ad6dd1cf7bba0f542d57baae:B 2024-11-21T21:15:08,300 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-21T21:15:08,302 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61499 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-21T21:15:08,302 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): 0357b2a5ad6dd1cf7bba0f542d57baae/C is initiating minor compaction (all files) 2024-11-21T21:15:08,302 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0357b2a5ad6dd1cf7bba0f542d57baae/C in TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:15:08,302 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/9cd1469b12684514a07d5e9a7ef4b49d, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/d38fabcec09d4b99a4debc3e5000e275, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/b7fce2561d034d03907788455623c07c, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/b188e5a622a94e9eb55d296655b58b50, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/0e98013a03ff4cbc94453c2a5006e4be] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp, totalSize=60.1 K 2024-11-21T21:15:08,303 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9cd1469b12684514a07d5e9a7ef4b49d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732223697503 2024-11-21T21:15:08,303 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting d38fabcec09d4b99a4debc3e5000e275, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732223699845 2024-11-21T21:15:08,303 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7fce2561d034d03907788455623c07c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732223701002 2024-11-21T21:15:08,303 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting b188e5a622a94e9eb55d296655b58b50, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732223703269 2024-11-21T21:15:08,304 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e98013a03ff4cbc94453c2a5006e4be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732223705434 2024-11-21T21:15:08,311 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121ac201d173dba40dfbd8db0d9b7cc0215_0357b2a5ad6dd1cf7bba0f542d57baae, store=[table=TestAcidGuarantees family=A region=0357b2a5ad6dd1cf7bba0f542d57baae] 2024-11-21T21:15:08,311 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ac201d173dba40dfbd8db0d9b7cc0215_0357b2a5ad6dd1cf7bba0f542d57baae because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0357b2a5ad6dd1cf7bba0f542d57baae] 2024-11-21T21:15:08,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742048_1224 (size=4469) 2024-11-21T21:15:08,319 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0357b2a5ad6dd1cf7bba0f542d57baae#C#compaction#191 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:08,320 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/a44e585aaef94731864ae02f1b7c8c3e is 50, key is test_row_0/C:col10/1732223706591/Put/seqid=0 2024-11-21T21:15:08,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742049_1225 (size=12915) 2024-11-21T21:15:08,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-21T21:15:08,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:15:08,364 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 0357b2a5ad6dd1cf7bba0f542d57baae 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-21T21:15:08,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0357b2a5ad6dd1cf7bba0f542d57baae, store=A 2024-11-21T21:15:08,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:08,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0357b2a5ad6dd1cf7bba0f542d57baae, store=B 2024-11-21T21:15:08,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:08,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0357b2a5ad6dd1cf7bba0f542d57baae, store=C 2024-11-21T21:15:08,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:08,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b978223f8fef489ca248f4194b9a3571_0357b2a5ad6dd1cf7bba0f542d57baae is 50, key is test_row_0/A:col10/1732223706638/Put/seqid=0 2024-11-21T21:15:08,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742050_1226 (size=12454) 2024-11-21T21:15:08,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T21:15:08,717 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0357b2a5ad6dd1cf7bba0f542d57baae#A#compaction#190 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:08,718 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/130b91768e244f8d8a542634176979ce is 175, key is test_row_0/A:col10/1732223706591/Put/seqid=0 2024-11-21T21:15:08,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742051_1227 (size=31869) 2024-11-21T21:15:08,762 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/a44e585aaef94731864ae02f1b7c8c3e as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/a44e585aaef94731864ae02f1b7c8c3e 2024-11-21T21:15:08,767 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 0357b2a5ad6dd1cf7bba0f542d57baae/C of 0357b2a5ad6dd1cf7bba0f542d57baae into a44e585aaef94731864ae02f1b7c8c3e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:08,767 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0357b2a5ad6dd1cf7bba0f542d57baae: 2024-11-21T21:15:08,767 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., storeName=0357b2a5ad6dd1cf7bba0f542d57baae/C, priority=11, startTime=1732223708262; duration=0sec 2024-11-21T21:15:08,767 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:08,767 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0357b2a5ad6dd1cf7bba0f542d57baae:C 2024-11-21T21:15:08,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:15:08,783 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b978223f8fef489ca248f4194b9a3571_0357b2a5ad6dd1cf7bba0f542d57baae to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b978223f8fef489ca248f4194b9a3571_0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:15:08,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/fbabb263c8a247da98d0ee42c7e01fcd, store: [table=TestAcidGuarantees family=A region=0357b2a5ad6dd1cf7bba0f542d57baae] 2024-11-21T21:15:08,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/fbabb263c8a247da98d0ee42c7e01fcd is 175, key is test_row_0/A:col10/1732223706638/Put/seqid=0 2024-11-21T21:15:08,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742052_1228 (size=31255) 2024-11-21T21:15:08,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:15:08,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. as already flushing 2024-11-21T21:15:08,802 INFO [Thread-732 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-732. 2024-11-21T21:15:08,802 INFO [Thread-736 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-736. 2024-11-21T21:15:08,802 DEBUG [Thread-732 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:15:08,802 DEBUG [Thread-732 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:08,802 DEBUG [Thread-736 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:15:08,802 DEBUG [Thread-736 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:08,802 DEBUG [Thread-736 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:08,802 DEBUG [Thread-732 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:08,802 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:15:08,803 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:15:09,169 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/130b91768e244f8d8a542634176979ce as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/130b91768e244f8d8a542634176979ce 2024-11-21T21:15:09,175 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 0357b2a5ad6dd1cf7bba0f542d57baae/A of 0357b2a5ad6dd1cf7bba0f542d57baae into 130b91768e244f8d8a542634176979ce(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:09,175 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0357b2a5ad6dd1cf7bba0f542d57baae: 2024-11-21T21:15:09,175 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., storeName=0357b2a5ad6dd1cf7bba0f542d57baae/A, priority=11, startTime=1732223708262; duration=0sec 2024-11-21T21:15:09,175 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:09,175 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0357b2a5ad6dd1cf7bba0f542d57baae:A 2024-11-21T21:15:09,190 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=313, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/fbabb263c8a247da98d0ee42c7e01fcd 2024-11-21T21:15:09,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/8374c122a76d4f1b9073e695e65247c3 is 50, key is test_row_0/B:col10/1732223706638/Put/seqid=0 2024-11-21T21:15:09,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742053_1229 (size=12301) 2024-11-21T21:15:09,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T21:15:09,604 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/8374c122a76d4f1b9073e695e65247c3 2024-11-21T21:15:09,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/8ec2b423f5d24187949153b161bd12e8 is 50, key is test_row_0/C:col10/1732223706638/Put/seqid=0 2024-11-21T21:15:09,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742054_1230 (size=12301) 2024-11-21T21:15:10,024 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/8ec2b423f5d24187949153b161bd12e8 2024-11-21T21:15:10,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/fbabb263c8a247da98d0ee42c7e01fcd as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/fbabb263c8a247da98d0ee42c7e01fcd 2024-11-21T21:15:10,039 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/fbabb263c8a247da98d0ee42c7e01fcd, entries=150, sequenceid=313, filesize=30.5 K 2024-11-21T21:15:10,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/8374c122a76d4f1b9073e695e65247c3 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/8374c122a76d4f1b9073e695e65247c3 2024-11-21T21:15:10,045 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/8374c122a76d4f1b9073e695e65247c3, entries=150, sequenceid=313, filesize=12.0 K 2024-11-21T21:15:10,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/8ec2b423f5d24187949153b161bd12e8 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/8ec2b423f5d24187949153b161bd12e8 2024-11-21T21:15:10,050 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/8ec2b423f5d24187949153b161bd12e8, entries=150, sequenceid=313, filesize=12.0 K 2024-11-21T21:15:10,051 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=13.42 KB/13740 for 0357b2a5ad6dd1cf7bba0f542d57baae in 1687ms, sequenceid=313, compaction requested=false 2024-11-21T21:15:10,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 0357b2a5ad6dd1cf7bba0f542d57baae: 2024-11-21T21:15:10,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:15:10,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-21T21:15:10,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-21T21:15:10,055 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-21T21:15:10,055 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7590 sec 2024-11-21T21:15:10,057 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 2.7640 sec 2024-11-21T21:15:11,032 INFO [Thread-738 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-738. 2024-11-21T21:15:11,032 DEBUG [Thread-738 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:15:11,032 DEBUG [Thread-738 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:11,032 DEBUG [Thread-738 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:11,033 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:15:11,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T21:15:11,462 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:15:11,603 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T21:15:14,435 INFO [Thread-734 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-734. 2024-11-21T21:15:14,435 DEBUG [Thread-734 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:15:14,435 DEBUG [Thread-734 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:14,435 DEBUG [Thread-734 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:14,435 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:15:14,495 INFO [Thread-740 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-740. 2024-11-21T21:15:14,496 DEBUG [Thread-740 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:15:14,496 DEBUG [Thread-740 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:14,496 DEBUG [Thread-740 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:14,496 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:15:14,497 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T21:15:14,497 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 119 2024-11-21T21:15:14,497 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 28 2024-11-21T21:15:14,497 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-21T21:15:14,497 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 21 2024-11-21T21:15:14,497 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 34 2024-11-21T21:15:14,497 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T21:15:14,497 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3066 2024-11-21T21:15:14,497 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2901 2024-11-21T21:15:14,497 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T21:15:14,497 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1281 2024-11-21T21:15:14,498 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3843 rows 2024-11-21T21:15:14,498 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1280 2024-11-21T21:15:14,498 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3834 rows 2024-11-21T21:15:14,498 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T21:15:14,498 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T21:15:14,498 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:14,499 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:14,499 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:15:14,500 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-21T21:15:14,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T21:15:14,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-21T21:15:14,504 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732223714504"}]},"ts":"1732223714504"} 2024-11-21T21:15:14,506 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T21:15:14,506 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T21:15:14,507 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T21:15:14,509 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0357b2a5ad6dd1cf7bba0f542d57baae, UNASSIGN}] 2024-11-21T21:15:14,510 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0357b2a5ad6dd1cf7bba0f542d57baae, UNASSIGN 2024-11-21T21:15:14,511 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=0357b2a5ad6dd1cf7bba0f542d57baae, regionState=CLOSING, regionLocation=849dee24a242,41213,1732223653469 2024-11-21T21:15:14,514 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0357b2a5ad6dd1cf7bba0f542d57baae, UNASSIGN because future has completed 2024-11-21T21:15:14,514 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T21:15:14,514 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469}] 2024-11-21T21:15:14,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-21T21:15:14,668 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(122): Close 0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:15:14,668 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T21:15:14,668 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1722): Closing 0357b2a5ad6dd1cf7bba0f542d57baae, disabling compactions & flushes 2024-11-21T21:15:14,668 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:15:14,668 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:15:14,669 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. after waiting 0 ms 2024-11-21T21:15:14,669 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:15:14,669 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(2902): Flushing 0357b2a5ad6dd1cf7bba0f542d57baae 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-21T21:15:14,670 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0357b2a5ad6dd1cf7bba0f542d57baae, store=A 2024-11-21T21:15:14,670 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:14,670 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0357b2a5ad6dd1cf7bba0f542d57baae, store=B 2024-11-21T21:15:14,670 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:14,670 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0357b2a5ad6dd1cf7bba0f542d57baae, store=C 2024-11-21T21:15:14,670 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:14,684 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411217e393f59c1dc43828ed9eba93b2a5f4c_0357b2a5ad6dd1cf7bba0f542d57baae is 50, key is test_row_0/A:col10/1732223714431/Put/seqid=0 2024-11-21T21:15:14,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742055_1231 (size=12454) 2024-11-21T21:15:14,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-21T21:15:15,090 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:15:15,102 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411217e393f59c1dc43828ed9eba93b2a5f4c_0357b2a5ad6dd1cf7bba0f542d57baae to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411217e393f59c1dc43828ed9eba93b2a5f4c_0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:15:15,105 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/fe41686ac7384f4986d9b0c737a4ee09, store: [table=TestAcidGuarantees family=A region=0357b2a5ad6dd1cf7bba0f542d57baae] 2024-11-21T21:15:15,106 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/fe41686ac7384f4986d9b0c737a4ee09 is 175, key is test_row_0/A:col10/1732223714431/Put/seqid=0 2024-11-21T21:15:15,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742056_1232 (size=31255) 2024-11-21T21:15:15,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-21T21:15:15,512 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=323, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/fe41686ac7384f4986d9b0c737a4ee09 2024-11-21T21:15:15,520 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/5331cab2cc3a4e70b2072c71cc76e8a1 is 50, key is test_row_0/B:col10/1732223714431/Put/seqid=0 2024-11-21T21:15:15,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742057_1233 (size=12301) 2024-11-21T21:15:15,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-21T21:15:15,925 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/5331cab2cc3a4e70b2072c71cc76e8a1 2024-11-21T21:15:15,942 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/493057e31a72418aa25649b2cbd2149c is 50, key is test_row_0/C:col10/1732223714431/Put/seqid=0 2024-11-21T21:15:15,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742058_1234 (size=12301) 2024-11-21T21:15:16,348 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/493057e31a72418aa25649b2cbd2149c 2024-11-21T21:15:16,359 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/A/fe41686ac7384f4986d9b0c737a4ee09 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/fe41686ac7384f4986d9b0c737a4ee09 2024-11-21T21:15:16,367 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/fe41686ac7384f4986d9b0c737a4ee09, entries=150, sequenceid=323, filesize=30.5 K 2024-11-21T21:15:16,368 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/B/5331cab2cc3a4e70b2072c71cc76e8a1 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/5331cab2cc3a4e70b2072c71cc76e8a1 2024-11-21T21:15:16,373 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/5331cab2cc3a4e70b2072c71cc76e8a1, entries=150, sequenceid=323, filesize=12.0 K 2024-11-21T21:15:16,374 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/.tmp/C/493057e31a72418aa25649b2cbd2149c as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/493057e31a72418aa25649b2cbd2149c 2024-11-21T21:15:16,379 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/493057e31a72418aa25649b2cbd2149c, entries=150, sequenceid=323, filesize=12.0 K 2024-11-21T21:15:16,380 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 0357b2a5ad6dd1cf7bba0f542d57baae in 1711ms, sequenceid=323, compaction requested=true 2024-11-21T21:15:16,381 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/aaaf623a271042bab6dbfa087fb5872c, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/37f5a2ac1c0f4a8593de2fb4730ae978, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/507143321ca44947aa9232c2d8538355, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/bc88da1a9324414193c161815e5bf91f, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/ca1e1974896f492ea4d8ebb6c41e6a2e, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/9da84e753020408788b2a0065a1d18e5, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/a859aca6d38146aaa073455f43b94e49, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/5ab2da46f1574bc492e8f81ae2e76f1f, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/3718785a08aa4faca3bf1eafd379d6cf, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/b196a54342b847409891cdd1ee8c7ce7, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/06c6d9c737f7465988009ed91fc05f41, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/d7449eef41884cf9b6b8a0b483dd69a7, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/aacc5067551b4e5b9f35f6fb55a5a170, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/100a6869126c4f388de925f23c1b8a51, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/37ac9d2ab5744a9080ad9b6a662bd215, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/4be429ae72a240aa8d2439591e90dee4, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/58d6b5dac12342f0926e190d2aaa87c6, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/ef8ad262ab9e4560908bfe47fd97017c] to archive 2024-11-21T21:15:16,382 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T21:15:16,385 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/aaaf623a271042bab6dbfa087fb5872c to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/aaaf623a271042bab6dbfa087fb5872c 2024-11-21T21:15:16,386 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/37f5a2ac1c0f4a8593de2fb4730ae978 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/37f5a2ac1c0f4a8593de2fb4730ae978 2024-11-21T21:15:16,388 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/507143321ca44947aa9232c2d8538355 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/507143321ca44947aa9232c2d8538355 2024-11-21T21:15:16,390 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/bc88da1a9324414193c161815e5bf91f to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/bc88da1a9324414193c161815e5bf91f 2024-11-21T21:15:16,392 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/ca1e1974896f492ea4d8ebb6c41e6a2e to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/ca1e1974896f492ea4d8ebb6c41e6a2e 2024-11-21T21:15:16,393 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/9da84e753020408788b2a0065a1d18e5 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/9da84e753020408788b2a0065a1d18e5 2024-11-21T21:15:16,394 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/a859aca6d38146aaa073455f43b94e49 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/a859aca6d38146aaa073455f43b94e49 2024-11-21T21:15:16,395 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/5ab2da46f1574bc492e8f81ae2e76f1f to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/5ab2da46f1574bc492e8f81ae2e76f1f 2024-11-21T21:15:16,396 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/3718785a08aa4faca3bf1eafd379d6cf to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/3718785a08aa4faca3bf1eafd379d6cf 2024-11-21T21:15:16,397 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/b196a54342b847409891cdd1ee8c7ce7 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/b196a54342b847409891cdd1ee8c7ce7 2024-11-21T21:15:16,399 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/06c6d9c737f7465988009ed91fc05f41 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/06c6d9c737f7465988009ed91fc05f41 2024-11-21T21:15:16,400 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/d7449eef41884cf9b6b8a0b483dd69a7 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/d7449eef41884cf9b6b8a0b483dd69a7 2024-11-21T21:15:16,401 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/aacc5067551b4e5b9f35f6fb55a5a170 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/aacc5067551b4e5b9f35f6fb55a5a170 2024-11-21T21:15:16,402 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/100a6869126c4f388de925f23c1b8a51 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/100a6869126c4f388de925f23c1b8a51 2024-11-21T21:15:16,402 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/37ac9d2ab5744a9080ad9b6a662bd215 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/37ac9d2ab5744a9080ad9b6a662bd215 2024-11-21T21:15:16,403 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/4be429ae72a240aa8d2439591e90dee4 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/4be429ae72a240aa8d2439591e90dee4 2024-11-21T21:15:16,404 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/58d6b5dac12342f0926e190d2aaa87c6 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/58d6b5dac12342f0926e190d2aaa87c6 2024-11-21T21:15:16,405 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/ef8ad262ab9e4560908bfe47fd97017c to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/ef8ad262ab9e4560908bfe47fd97017c 2024-11-21T21:15:16,406 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/b4ba571104aa4047bb122ad27ef72bdc, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/87526025ca074be0882e55cea07fc2a8, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/4fb93c62d4fb40dd97b51d5a34dcec8e, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/fdde0fdfcf034e7fb2d03714815ef40d, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/8144e25ce95c4f7d8d1cb5940adad345, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/cd9c6020491c4a73ad896307eaac6d43, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/7a44c5f86fb44278bf874cf6100b2b5c, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/5d0408c402574537931e3c7dd08decea, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/1464321b38a044878435b31a25d255a9, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/e280f96e13b144df92ec47b8c676a5bc, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/e069054bcea44b9f9d86c59b92a4c880, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/ea13f49e50c84e2abf5af4afbcc88552, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/a897c6ac9d5b4f1884ffda8c866b17b1, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/bb40aa3a0163476da6bfb16ca13ed9b6, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/d686f35903d0455fbf41aba322a8b897, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/d2f95036e9d645118d7cd355670e5518, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/874795ad81e14853aae67b05614eef19, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/a5c1e10c974a41e083f6ec7f2abcdec2] to archive 2024-11-21T21:15:16,407 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T21:15:16,408 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/b4ba571104aa4047bb122ad27ef72bdc to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/b4ba571104aa4047bb122ad27ef72bdc 2024-11-21T21:15:16,409 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/87526025ca074be0882e55cea07fc2a8 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/87526025ca074be0882e55cea07fc2a8 2024-11-21T21:15:16,410 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/4fb93c62d4fb40dd97b51d5a34dcec8e to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/4fb93c62d4fb40dd97b51d5a34dcec8e 2024-11-21T21:15:16,410 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/fdde0fdfcf034e7fb2d03714815ef40d to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/fdde0fdfcf034e7fb2d03714815ef40d 2024-11-21T21:15:16,411 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/8144e25ce95c4f7d8d1cb5940adad345 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/8144e25ce95c4f7d8d1cb5940adad345 2024-11-21T21:15:16,412 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/cd9c6020491c4a73ad896307eaac6d43 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/cd9c6020491c4a73ad896307eaac6d43 2024-11-21T21:15:16,413 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/7a44c5f86fb44278bf874cf6100b2b5c to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/7a44c5f86fb44278bf874cf6100b2b5c 2024-11-21T21:15:16,414 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/5d0408c402574537931e3c7dd08decea to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/5d0408c402574537931e3c7dd08decea 2024-11-21T21:15:16,414 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/1464321b38a044878435b31a25d255a9 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/1464321b38a044878435b31a25d255a9 2024-11-21T21:15:16,415 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/e280f96e13b144df92ec47b8c676a5bc to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/e280f96e13b144df92ec47b8c676a5bc 2024-11-21T21:15:16,416 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/e069054bcea44b9f9d86c59b92a4c880 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/e069054bcea44b9f9d86c59b92a4c880 2024-11-21T21:15:16,417 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/ea13f49e50c84e2abf5af4afbcc88552 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/ea13f49e50c84e2abf5af4afbcc88552 2024-11-21T21:15:16,418 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/a897c6ac9d5b4f1884ffda8c866b17b1 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/a897c6ac9d5b4f1884ffda8c866b17b1 2024-11-21T21:15:16,419 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/bb40aa3a0163476da6bfb16ca13ed9b6 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/bb40aa3a0163476da6bfb16ca13ed9b6 2024-11-21T21:15:16,420 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/d686f35903d0455fbf41aba322a8b897 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/d686f35903d0455fbf41aba322a8b897 2024-11-21T21:15:16,421 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/d2f95036e9d645118d7cd355670e5518 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/d2f95036e9d645118d7cd355670e5518 2024-11-21T21:15:16,422 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/874795ad81e14853aae67b05614eef19 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/874795ad81e14853aae67b05614eef19 2024-11-21T21:15:16,422 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/a5c1e10c974a41e083f6ec7f2abcdec2 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/a5c1e10c974a41e083f6ec7f2abcdec2 2024-11-21T21:15:16,424 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/a0db3d97b2fa4ef2a44b3c119dd5d6f6, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/3ab1ed8067c0434d951ac2712ff059e6, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/1d322b6eaa16455590f35a46d059109f, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/600c72b442eb4b84b5229905d93a414a, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/c428e4d5eeed47998cc8e75ae4b1e874, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/ad1a008a471c4807a5fda9e830abf337, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/b1315c76ddf940138a2b93c4a3e590ae, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/5656a04d399049ad9d36acf1c10d306b, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/aeecd218b3e84bfa9bee443c2bc1f6bc, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/d087ac4f72144d43bb1fb83979f274a2, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/6ab43bcf58724f94bddbf1a71af364da, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/1c2bb4d66e994bc9b886f2deff40a393, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/9cd1469b12684514a07d5e9a7ef4b49d, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/7d340644b0304c60993ffb395d5eacb2, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/d38fabcec09d4b99a4debc3e5000e275, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/b7fce2561d034d03907788455623c07c, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/b188e5a622a94e9eb55d296655b58b50, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/0e98013a03ff4cbc94453c2a5006e4be] to archive 2024-11-21T21:15:16,424 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T21:15:16,425 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/a0db3d97b2fa4ef2a44b3c119dd5d6f6 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/a0db3d97b2fa4ef2a44b3c119dd5d6f6 2024-11-21T21:15:16,426 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/3ab1ed8067c0434d951ac2712ff059e6 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/3ab1ed8067c0434d951ac2712ff059e6 2024-11-21T21:15:16,427 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/1d322b6eaa16455590f35a46d059109f to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/1d322b6eaa16455590f35a46d059109f 2024-11-21T21:15:16,427 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/600c72b442eb4b84b5229905d93a414a to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/600c72b442eb4b84b5229905d93a414a 2024-11-21T21:15:16,428 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/c428e4d5eeed47998cc8e75ae4b1e874 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/c428e4d5eeed47998cc8e75ae4b1e874 2024-11-21T21:15:16,429 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/ad1a008a471c4807a5fda9e830abf337 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/ad1a008a471c4807a5fda9e830abf337 2024-11-21T21:15:16,430 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/b1315c76ddf940138a2b93c4a3e590ae to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/b1315c76ddf940138a2b93c4a3e590ae 2024-11-21T21:15:16,430 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/5656a04d399049ad9d36acf1c10d306b to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/5656a04d399049ad9d36acf1c10d306b 2024-11-21T21:15:16,431 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/aeecd218b3e84bfa9bee443c2bc1f6bc to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/aeecd218b3e84bfa9bee443c2bc1f6bc 2024-11-21T21:15:16,432 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/d087ac4f72144d43bb1fb83979f274a2 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/d087ac4f72144d43bb1fb83979f274a2 2024-11-21T21:15:16,433 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/6ab43bcf58724f94bddbf1a71af364da to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/6ab43bcf58724f94bddbf1a71af364da 2024-11-21T21:15:16,434 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/1c2bb4d66e994bc9b886f2deff40a393 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/1c2bb4d66e994bc9b886f2deff40a393 2024-11-21T21:15:16,435 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/9cd1469b12684514a07d5e9a7ef4b49d to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/9cd1469b12684514a07d5e9a7ef4b49d 2024-11-21T21:15:16,435 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/7d340644b0304c60993ffb395d5eacb2 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/7d340644b0304c60993ffb395d5eacb2 2024-11-21T21:15:16,436 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/d38fabcec09d4b99a4debc3e5000e275 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/d38fabcec09d4b99a4debc3e5000e275 2024-11-21T21:15:16,437 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/b7fce2561d034d03907788455623c07c to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/b7fce2561d034d03907788455623c07c 2024-11-21T21:15:16,438 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/b188e5a622a94e9eb55d296655b58b50 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/b188e5a622a94e9eb55d296655b58b50 2024-11-21T21:15:16,439 DEBUG [StoreCloser-TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/0e98013a03ff4cbc94453c2a5006e4be to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/0e98013a03ff4cbc94453c2a5006e4be 2024-11-21T21:15:16,442 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/recovered.edits/326.seqid, newMaxSeqId=326, maxSeqId=4 2024-11-21T21:15:16,443 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. 2024-11-21T21:15:16,443 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1676): Region close journal for 0357b2a5ad6dd1cf7bba0f542d57baae: Waiting for close lock at 1732223714668Running coprocessor pre-close hooks at 1732223714668Disabling compacts and flushes for region at 1732223714668Disabling writes for close at 1732223714669 (+1 ms)Obtaining lock to block concurrent updates at 1732223714669Preparing flush snapshotting stores in 0357b2a5ad6dd1cf7bba0f542d57baae at 1732223714669Finished memstore snapshotting TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732223714670 (+1 ms)Flushing stores of TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae. at 1732223714672 (+2 ms)Flushing 0357b2a5ad6dd1cf7bba0f542d57baae/A: creating writer at 1732223714672Flushing mob file 0357b2a5ad6dd1cf7bba0f542d57baae/A: appending metadata at 1732223714684 (+12 ms)Flushing mob file 0357b2a5ad6dd1cf7bba0f542d57baae/A: closing flushed file at 1732223714684Flushing 0357b2a5ad6dd1cf7bba0f542d57baae/A: appending metadata at 1732223715105 (+421 ms)Flushing 0357b2a5ad6dd1cf7bba0f542d57baae/A: closing flushed file at 1732223715105Flushing 0357b2a5ad6dd1cf7bba0f542d57baae/B: creating writer at 1732223715516 (+411 ms)Flushing 0357b2a5ad6dd1cf7bba0f542d57baae/B: appending metadata at 1732223715520 (+4 ms)Flushing 0357b2a5ad6dd1cf7bba0f542d57baae/B: closing flushed file at 1732223715520Flushing 0357b2a5ad6dd1cf7bba0f542d57baae/C: creating writer at 1732223715936 (+416 ms)Flushing 0357b2a5ad6dd1cf7bba0f542d57baae/C: appending metadata at 1732223715941 (+5 ms)Flushing 0357b2a5ad6dd1cf7bba0f542d57baae/C: closing flushed file at 1732223715941Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d7388c3: reopening flushed file at 1732223716358 (+417 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ee42e8f: reopening flushed file at 1732223716367 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1be4d665: reopening flushed file at 1732223716373 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 0357b2a5ad6dd1cf7bba0f542d57baae in 1711ms, sequenceid=323, compaction requested=true at 1732223716380 (+7 ms)Writing region close event to WAL at 1732223716440 (+60 ms)Running coprocessor post-close hooks at 1732223716443 (+3 ms)Closed at 1732223716443 2024-11-21T21:15:16,445 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(157): Closed 0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:15:16,445 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=0357b2a5ad6dd1cf7bba0f542d57baae, regionState=CLOSED 2024-11-21T21:15:16,446 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 because future has completed 2024-11-21T21:15:16,448 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-21T21:15:16,448 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseRegionProcedure 0357b2a5ad6dd1cf7bba0f542d57baae, server=849dee24a242,41213,1732223653469 in 1.9330 sec 2024-11-21T21:15:16,450 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-21T21:15:16,450 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0357b2a5ad6dd1cf7bba0f542d57baae, UNASSIGN in 1.9390 sec 2024-11-21T21:15:16,451 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-21T21:15:16,451 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9430 sec 2024-11-21T21:15:16,452 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732223716452"}]},"ts":"1732223716452"} 2024-11-21T21:15:16,453 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T21:15:16,453 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T21:15:16,455 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9540 sec 2024-11-21T21:15:16,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-21T21:15:16,650 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:15:16,651 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-21T21:15:16,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:15:16,653 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:15:16,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-21T21:15:16,653 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=60, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:15:16,656 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:15:16,659 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A, FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B, FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C, FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/recovered.edits] 2024-11-21T21:15:16,662 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/130b91768e244f8d8a542634176979ce to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/130b91768e244f8d8a542634176979ce 2024-11-21T21:15:16,663 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/fbabb263c8a247da98d0ee42c7e01fcd to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/fbabb263c8a247da98d0ee42c7e01fcd 2024-11-21T21:15:16,664 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/fe41686ac7384f4986d9b0c737a4ee09 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/A/fe41686ac7384f4986d9b0c737a4ee09 2024-11-21T21:15:16,666 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/5331cab2cc3a4e70b2072c71cc76e8a1 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/5331cab2cc3a4e70b2072c71cc76e8a1 2024-11-21T21:15:16,667 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/8374c122a76d4f1b9073e695e65247c3 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/8374c122a76d4f1b9073e695e65247c3 2024-11-21T21:15:16,668 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/bf24f162516a4a628ba762d68b6fef91 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/B/bf24f162516a4a628ba762d68b6fef91 2024-11-21T21:15:16,670 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/493057e31a72418aa25649b2cbd2149c to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/493057e31a72418aa25649b2cbd2149c 2024-11-21T21:15:16,671 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/8ec2b423f5d24187949153b161bd12e8 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/8ec2b423f5d24187949153b161bd12e8 2024-11-21T21:15:16,672 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/a44e585aaef94731864ae02f1b7c8c3e to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/C/a44e585aaef94731864ae02f1b7c8c3e 2024-11-21T21:15:16,675 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/recovered.edits/326.seqid to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae/recovered.edits/326.seqid 2024-11-21T21:15:16,675 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:15:16,675 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T21:15:16,675 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T21:15:16,676 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-21T21:15:16,680 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121226e63ae98e346d993694c711faa5ce4_0357b2a5ad6dd1cf7bba0f542d57baae to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121226e63ae98e346d993694c711faa5ce4_0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:15:16,681 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121498e6787b1b448a296e7322585269e03_0357b2a5ad6dd1cf7bba0f542d57baae to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121498e6787b1b448a296e7322585269e03_0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:15:16,682 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215424bae0684947fe96253dcb6ab7bf49_0357b2a5ad6dd1cf7bba0f542d57baae to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215424bae0684947fe96253dcb6ab7bf49_0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:15:16,683 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112156543518382c47fd91c37e260399b2ed_0357b2a5ad6dd1cf7bba0f542d57baae to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112156543518382c47fd91c37e260399b2ed_0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:15:16,684 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215eca576c22754fb8a037e3a0070cf1bb_0357b2a5ad6dd1cf7bba0f542d57baae to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215eca576c22754fb8a037e3a0070cf1bb_0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:15:16,685 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112163b22fef24dd4801a4d15d2dbfd77dd1_0357b2a5ad6dd1cf7bba0f542d57baae to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112163b22fef24dd4801a4d15d2dbfd77dd1_0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:15:16,687 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411217e393f59c1dc43828ed9eba93b2a5f4c_0357b2a5ad6dd1cf7bba0f542d57baae to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411217e393f59c1dc43828ed9eba93b2a5f4c_0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:15:16,688 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112186ccfc30eca14c6db0c99cd40c6de61c_0357b2a5ad6dd1cf7bba0f542d57baae to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112186ccfc30eca14c6db0c99cd40c6de61c_0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:15:16,689 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121990d8d3c710a4d84a100e8b2a8f1eba5_0357b2a5ad6dd1cf7bba0f542d57baae to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121990d8d3c710a4d84a100e8b2a8f1eba5_0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:15:16,690 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219a90a7ec10b4459881640c8874c62d6a_0357b2a5ad6dd1cf7bba0f542d57baae to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219a90a7ec10b4459881640c8874c62d6a_0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:15:16,691 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219d77811cf81148b489be6dc79a43d483_0357b2a5ad6dd1cf7bba0f542d57baae to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219d77811cf81148b489be6dc79a43d483_0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:15:16,693 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ab19cd5a0d904dd7aedffa91b034c183_0357b2a5ad6dd1cf7bba0f542d57baae to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ab19cd5a0d904dd7aedffa91b034c183_0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:15:16,694 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b978223f8fef489ca248f4194b9a3571_0357b2a5ad6dd1cf7bba0f542d57baae to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b978223f8fef489ca248f4194b9a3571_0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:15:16,695 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c6085d10aebe477b8c056b070c12f5f5_0357b2a5ad6dd1cf7bba0f542d57baae to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c6085d10aebe477b8c056b070c12f5f5_0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:15:16,696 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d62ba46cb2b149ceb139a7ef077f31ea_0357b2a5ad6dd1cf7bba0f542d57baae to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d62ba46cb2b149ceb139a7ef077f31ea_0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:15:16,697 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121fe84be96f983487c86692d4d29265d2e_0357b2a5ad6dd1cf7bba0f542d57baae to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121fe84be96f983487c86692d4d29265d2e_0357b2a5ad6dd1cf7bba0f542d57baae 2024-11-21T21:15:16,698 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T21:15:16,700 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=60, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:15:16,702 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T21:15:16,704 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T21:15:16,705 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=60, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:15:16,705 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T21:15:16,705 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732223716705"}]},"ts":"9223372036854775807"} 2024-11-21T21:15:16,707 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T21:15:16,707 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 0357b2a5ad6dd1cf7bba0f542d57baae, NAME => 'TestAcidGuarantees,,1732223685049.0357b2a5ad6dd1cf7bba0f542d57baae.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T21:15:16,708 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T21:15:16,708 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732223716708"}]},"ts":"9223372036854775807"} 2024-11-21T21:15:16,709 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T21:15:16,710 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=60, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:15:16,711 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 59 msec 2024-11-21T21:15:16,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-21T21:15:16,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T21:15:16,761 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:15:16,769 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 250), OpenFileDescriptor=453 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=341 (was 370), ProcessCount=11 (was 11), AvailableMemoryMB=11484 (was 11633) 2024-11-21T21:15:16,777 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=341, ProcessCount=11, AvailableMemoryMB=11484 2024-11-21T21:15:16,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T21:15:16,779 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T21:15:16,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T21:15:16,781 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T21:15:16,781 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:15:16,781 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 61 2024-11-21T21:15:16,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-21T21:15:16,782 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T21:15:16,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742059_1235 (size=1061) 2024-11-21T21:15:16,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-21T21:15:17,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-21T21:15:17,194 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f 2024-11-21T21:15:17,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742060_1236 (size=53) 2024-11-21T21:15:17,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-21T21:15:17,608 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:15:17,608 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 4a01a47e5e966934386d5c50a408380a, disabling compactions & flushes 2024-11-21T21:15:17,608 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:17,608 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:17,608 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. after waiting 0 ms 2024-11-21T21:15:17,608 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:17,608 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:17,609 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4a01a47e5e966934386d5c50a408380a: Waiting for close lock at 1732223717608Disabling compacts and flushes for region at 1732223717608Disabling writes for close at 1732223717608Writing region close event to WAL at 1732223717608Closed at 1732223717608 2024-11-21T21:15:17,611 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T21:15:17,612 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732223717611"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732223717611"}]},"ts":"1732223717611"} 2024-11-21T21:15:17,614 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T21:15:17,615 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T21:15:17,616 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732223717615"}]},"ts":"1732223717615"} 2024-11-21T21:15:17,617 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T21:15:17,618 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4a01a47e5e966934386d5c50a408380a, ASSIGN}] 2024-11-21T21:15:17,619 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4a01a47e5e966934386d5c50a408380a, ASSIGN 2024-11-21T21:15:17,620 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4a01a47e5e966934386d5c50a408380a, ASSIGN; state=OFFLINE, location=849dee24a242,41213,1732223653469; forceNewPlan=false, retain=false 2024-11-21T21:15:17,771 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=4a01a47e5e966934386d5c50a408380a, regionState=OPENING, regionLocation=849dee24a242,41213,1732223653469 2024-11-21T21:15:17,776 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4a01a47e5e966934386d5c50a408380a, ASSIGN because future has completed 2024-11-21T21:15:17,777 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469}] 2024-11-21T21:15:17,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-21T21:15:17,938 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:17,938 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7752): Opening region: {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} 2024-11-21T21:15:17,939 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4a01a47e5e966934386d5c50a408380a 2024-11-21T21:15:17,939 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:15:17,939 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7794): checking encryption for 4a01a47e5e966934386d5c50a408380a 2024-11-21T21:15:17,939 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7797): checking classloading for 4a01a47e5e966934386d5c50a408380a 2024-11-21T21:15:17,942 INFO [StoreOpener-4a01a47e5e966934386d5c50a408380a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4a01a47e5e966934386d5c50a408380a 2024-11-21T21:15:17,944 INFO [StoreOpener-4a01a47e5e966934386d5c50a408380a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T21:15:17,945 INFO [StoreOpener-4a01a47e5e966934386d5c50a408380a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4a01a47e5e966934386d5c50a408380a columnFamilyName A 2024-11-21T21:15:17,945 DEBUG [StoreOpener-4a01a47e5e966934386d5c50a408380a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:15:17,946 INFO [StoreOpener-4a01a47e5e966934386d5c50a408380a-1 {}] regionserver.HStore(327): Store=4a01a47e5e966934386d5c50a408380a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:15:17,946 INFO [StoreOpener-4a01a47e5e966934386d5c50a408380a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4a01a47e5e966934386d5c50a408380a 2024-11-21T21:15:17,948 INFO [StoreOpener-4a01a47e5e966934386d5c50a408380a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T21:15:17,948 INFO [StoreOpener-4a01a47e5e966934386d5c50a408380a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4a01a47e5e966934386d5c50a408380a columnFamilyName B 2024-11-21T21:15:17,948 DEBUG [StoreOpener-4a01a47e5e966934386d5c50a408380a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:15:17,949 INFO [StoreOpener-4a01a47e5e966934386d5c50a408380a-1 {}] regionserver.HStore(327): Store=4a01a47e5e966934386d5c50a408380a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:15:17,950 INFO [StoreOpener-4a01a47e5e966934386d5c50a408380a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4a01a47e5e966934386d5c50a408380a 2024-11-21T21:15:17,951 INFO [StoreOpener-4a01a47e5e966934386d5c50a408380a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T21:15:17,951 INFO [StoreOpener-4a01a47e5e966934386d5c50a408380a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4a01a47e5e966934386d5c50a408380a columnFamilyName C 2024-11-21T21:15:17,951 DEBUG [StoreOpener-4a01a47e5e966934386d5c50a408380a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:15:17,952 INFO [StoreOpener-4a01a47e5e966934386d5c50a408380a-1 {}] regionserver.HStore(327): Store=4a01a47e5e966934386d5c50a408380a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:15:17,952 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:17,952 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1038): replaying wal for 4a01a47e5e966934386d5c50a408380a 2024-11-21T21:15:17,953 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a 2024-11-21T21:15:17,953 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a 2024-11-21T21:15:17,954 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1048): stopping wal replay for 4a01a47e5e966934386d5c50a408380a 2024-11-21T21:15:17,955 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1060): Cleaning up temporary data for 4a01a47e5e966934386d5c50a408380a 2024-11-21T21:15:17,955 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T21:15:17,957 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1093): writing seq id for 4a01a47e5e966934386d5c50a408380a 2024-11-21T21:15:17,959 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T21:15:17,959 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1114): Opened 4a01a47e5e966934386d5c50a408380a; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60234481, jitterRate=-0.10243628919124603}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T21:15:17,959 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4a01a47e5e966934386d5c50a408380a 2024-11-21T21:15:17,960 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1006): Region open journal for 4a01a47e5e966934386d5c50a408380a: Running coprocessor pre-open hook at 1732223717940Writing region info on filesystem at 1732223717940Initializing all the Stores at 1732223717941 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732223717941Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732223717942 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732223717942Cleaning up temporary data from old regions at 1732223717955 (+13 ms)Running coprocessor post-open hooks at 1732223717959 (+4 ms)Region opened successfully at 1732223717960 (+1 ms) 2024-11-21T21:15:17,962 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., pid=63, masterSystemTime=1732223717931 2024-11-21T21:15:17,964 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:17,964 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:17,965 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=4a01a47e5e966934386d5c50a408380a, regionState=OPEN, openSeqNum=2, regionLocation=849dee24a242,41213,1732223653469 2024-11-21T21:15:17,966 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 because future has completed 2024-11-21T21:15:17,968 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-21T21:15:17,968 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 in 191 msec 2024-11-21T21:15:17,970 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-21T21:15:17,970 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4a01a47e5e966934386d5c50a408380a, ASSIGN in 350 msec 2024-11-21T21:15:17,971 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T21:15:17,971 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732223717971"}]},"ts":"1732223717971"} 2024-11-21T21:15:17,972 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T21:15:17,973 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T21:15:17,974 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1940 sec 2024-11-21T21:15:18,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-21T21:15:18,930 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:15:18,932 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@781611af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:18,932 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:15:18,932 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:15:18,933 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:15:18,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:15:18,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:15:18,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c612f6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:18,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:15:18,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:15:18,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:18,935 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34200, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:15:18,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21d0b352, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:18,936 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:15:18,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:15:18,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:15:18,938 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53160, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:15:18,939 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T21:15:18,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 849dee24a242,42485,1732223652547 2024-11-21T21:15:18,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@17824b2f 2024-11-21T21:15:18,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T21:15:18,943 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34208, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T21:15:18,944 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32f1f4e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:18,944 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:15:18,945 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:15:18,945 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:15:18,945 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:15:18,945 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:15:18,946 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f064478, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:18,946 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:15:18,946 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:15:18,946 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:18,947 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34222, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:15:18,947 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@de7b0d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:18,948 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cf99f0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:18,948 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:15:18,948 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:15:18,949 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:15:18,949 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:15:18,949 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:15:18,949 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c823717, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:18,949 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:15:18,949 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:15:18,949 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:18,950 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34240, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:15:18,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@275fe932, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:18,951 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6594bb9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:18,951 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:15:18,951 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:15:18,952 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:15:18,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:15:18,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:15:18,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bdc1fca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:18,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:15:18,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:15:18,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:18,953 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34258, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:15:18,954 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77d30d4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:18,954 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61deb417, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:18,954 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:15:18,954 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:15:18,955 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:15:18,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:15:18,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:15:18,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cdfd353, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:18,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:15:18,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:15:18,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:18,956 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34280, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:15:18,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76b100a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:18,957 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47f17e9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:18,957 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:15:18,958 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:15:18,958 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:15:18,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:15:18,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:15:18,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@572ecb0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:18,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:15:18,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:15:18,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:18,959 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34288, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:15:18,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d0e4a20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:18,960 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7acbee0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:18,960 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:15:18,961 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:15:18,961 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:15:18,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:15:18,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:15:18,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59cf2321, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:18,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:15:18,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:15:18,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:18,962 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34298, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:15:18,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b290969, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:18,963 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46eaeeea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:18,963 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:15:18,963 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:15:18,964 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:15:18,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:15:18,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:15:18,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c8e6927, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:18,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:15:18,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:15:18,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:18,965 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34320, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:15:18,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f970279, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:18,966 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bee866e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:18,966 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:15:18,966 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:15:18,967 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:15:18,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:15:18,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:15:18,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f174051, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:18,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:15:18,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:15:18,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:18,968 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34342, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:15:18,969 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b311307, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:18,969 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@148ac527, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:18,969 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:15:18,969 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:15:18,970 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:15:18,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:15:18,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:15:18,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77da3dac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:18,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:15:18,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:15:18,971 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:18,971 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34368, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:15:18,972 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1eba6d96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:18,972 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4199deaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:18,972 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:15:18,972 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:15:18,973 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:15:18,973 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:15:18,973 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:15:18,973 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53ed344f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:18,973 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:15:18,973 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:15:18,973 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:18,974 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34396, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:15:18,974 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2613faad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:18,975 DEBUG [Thread-1106 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:15:18,975 DEBUG [Thread-1123 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:15:18,975 DEBUG [Thread-1112 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:15:18,975 DEBUG [Thread-1108 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:15:18,976 DEBUG [Thread-1121 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:15:18,976 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:15:18,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:15:18,976 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:15:18,976 DEBUG [Thread-1119 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:15:18,976 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:15:18,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:15:18,976 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:15:18,976 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:15:18,976 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:15:18,976 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:15:18,977 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:15:18,977 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:15:18,977 DEBUG [Thread-1104 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:15:18,977 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:15:18,978 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:15:18,978 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:15:18,978 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53198, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:15:18,978 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:15:18,978 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53216, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:15:18,978 DEBUG [Thread-1110 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:15:18,978 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53168, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:15:18,978 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:15:18,978 DEBUG [Thread-1117 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:15:18,978 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53214, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:15:18,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-21T21:15:18,979 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53218, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:15:18,979 DEBUG [Thread-1115 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:15:18,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:15:18,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:15:18,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:15:18,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:15:18,981 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53220, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:15:18,981 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:15:18,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2] 2024-11-21T21:15:18,982 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:15:18,982 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:15:18,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2] 2024-11-21T21:15:18,983 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53244, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:15:18,983 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53234, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:15:18,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2] 2024-11-21T21:15:18,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:15:18,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2] 2024-11-21T21:15:18,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:15:18,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2] 2024-11-21T21:15:18,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2] 2024-11-21T21:15:18,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-21T21:15:18,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2] 2024-11-21T21:15:18,988 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53256, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:15:18,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2] 2024-11-21T21:15:18,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2] 2024-11-21T21:15:18,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2] 2024-11-21T21:15:19,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 4a01a47e5e966934386d5c50a408380a 2024-11-21T21:15:19,004 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a01a47e5e966934386d5c50a408380a 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-21T21:15:19,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=A 2024-11-21T21:15:19,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:19,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=B 2024-11-21T21:15:19,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:19,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=C 2024-11-21T21:15:19,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:19,037 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/a455f76a627548e7b993e7e3547668d2 is 50, key is test_row_0/A:col10/1732223718986/Put/seqid=0 2024-11-21T21:15:19,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:19,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223779034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:19,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:19,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223779036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:19,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:19,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:19,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:19,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:19,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:19,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:19,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:19,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223779036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:19,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:19,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:19,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:19,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:19,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:19,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223779039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:19,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223779042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:19,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:19,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:19,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:19,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:19,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:19,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:19,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742061_1237 (size=12001) 2024-11-21T21:15:19,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-21T21:15:19,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-21T21:15:19,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:19,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:19,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:19,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:19,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:19,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:19,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:19,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223779150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:19,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:19,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223779151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:19,155 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:19,155 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:19,155 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:19,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:19,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:19,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:19,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:19,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:19,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223779151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:19,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223779151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:19,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:19,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:19,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:19,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:19,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:19,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:19,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:19,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223779151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:19,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:19,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:19,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:19,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-21T21:15:19,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:19,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:19,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:19,296 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:19,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:19,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:19,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-21T21:15:19,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:19,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223779360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:19,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:19,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:19,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223779360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:19,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:19,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:19,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:19,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:19,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:19,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:19,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223779360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:19,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:19,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:19,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:19,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:19,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223779360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:19,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:19,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:19,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:19,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:19,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223779360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:19,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:19,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:19,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:19,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-21T21:15:19,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:19,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:19,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:19,450 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:19,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:19,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:19,456 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/a455f76a627548e7b993e7e3547668d2 2024-11-21T21:15:19,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/9eecf6bf0d0b424ab50005316b35b16c is 50, key is test_row_0/B:col10/1732223718986/Put/seqid=0 2024-11-21T21:15:19,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742062_1238 (size=12001) 2024-11-21T21:15:19,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-21T21:15:19,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:19,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:19,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:19,604 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:19,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:19,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:19,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-21T21:15:19,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:19,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223779670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:19,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:19,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:19,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:19,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:19,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:19,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223779670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:19,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223779670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:19,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:19,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:19,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:19,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:19,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:19,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:19,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:19,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223779671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:19,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223779671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:19,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:19,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:19,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:19,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:19,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:19,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:19,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:19,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-21T21:15:19,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:19,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:19,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:19,759 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:19,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:19,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:19,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-21T21:15:19,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:19,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:19,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:19,912 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:19,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:19,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:19,917 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/9eecf6bf0d0b424ab50005316b35b16c 2024-11-21T21:15:19,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/e512e73b5f35445eaddfcfb9e3ad1691 is 50, key is test_row_0/C:col10/1732223718986/Put/seqid=0 2024-11-21T21:15:19,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742063_1239 (size=12001) 2024-11-21T21:15:19,995 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/e512e73b5f35445eaddfcfb9e3ad1691 2024-11-21T21:15:20,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/a455f76a627548e7b993e7e3547668d2 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/a455f76a627548e7b993e7e3547668d2 2024-11-21T21:15:20,011 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/a455f76a627548e7b993e7e3547668d2, entries=150, sequenceid=14, filesize=11.7 K 2024-11-21T21:15:20,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/9eecf6bf0d0b424ab50005316b35b16c as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/9eecf6bf0d0b424ab50005316b35b16c 2024-11-21T21:15:20,020 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/9eecf6bf0d0b424ab50005316b35b16c, entries=150, sequenceid=14, filesize=11.7 K 2024-11-21T21:15:20,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/e512e73b5f35445eaddfcfb9e3ad1691 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/e512e73b5f35445eaddfcfb9e3ad1691 2024-11-21T21:15:20,032 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/e512e73b5f35445eaddfcfb9e3ad1691, entries=150, sequenceid=14, filesize=11.7 K 2024-11-21T21:15:20,033 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 4a01a47e5e966934386d5c50a408380a in 1029ms, sequenceid=14, compaction requested=false 2024-11-21T21:15:20,033 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-21T21:15:20,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:20,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-21T21:15:20,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:20,066 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 4a01a47e5e966934386d5c50a408380a 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-21T21:15:20,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=A 2024-11-21T21:15:20,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:20,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=B 2024-11-21T21:15:20,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:20,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=C 2024-11-21T21:15:20,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:20,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/9048cd595602481bb4fe7f0922900b3b is 50, key is test_row_0/A:col10/1732223719038/Put/seqid=0 2024-11-21T21:15:20,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742064_1240 (size=12001) 2024-11-21T21:15:20,095 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/9048cd595602481bb4fe7f0922900b3b 2024-11-21T21:15:20,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/b445f33fb2b3411e90ad7684d7c7f2f2 is 50, key is test_row_0/B:col10/1732223719038/Put/seqid=0 2024-11-21T21:15:20,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-21T21:15:20,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742065_1241 (size=12001) 2024-11-21T21:15:20,142 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/b445f33fb2b3411e90ad7684d7c7f2f2 2024-11-21T21:15:20,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/adbdbe7c41494e6f87a29c5dfa0a67a8 is 50, key is test_row_0/C:col10/1732223719038/Put/seqid=0 2024-11-21T21:15:20,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 4a01a47e5e966934386d5c50a408380a 2024-11-21T21:15:20,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:20,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742066_1242 (size=12001) 2024-11-21T21:15:20,185 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/adbdbe7c41494e6f87a29c5dfa0a67a8 2024-11-21T21:15:20,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/9048cd595602481bb4fe7f0922900b3b as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/9048cd595602481bb4fe7f0922900b3b 2024-11-21T21:15:20,198 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/9048cd595602481bb4fe7f0922900b3b, entries=150, sequenceid=33, filesize=11.7 K 2024-11-21T21:15:20,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/b445f33fb2b3411e90ad7684d7c7f2f2 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/b445f33fb2b3411e90ad7684d7c7f2f2 2024-11-21T21:15:20,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:20,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223780198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:20,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:20,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:20,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223780198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:20,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:20,205 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/b445f33fb2b3411e90ad7684d7c7f2f2, entries=150, sequenceid=33, filesize=11.7 K 2024-11-21T21:15:20,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/adbdbe7c41494e6f87a29c5dfa0a67a8 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/adbdbe7c41494e6f87a29c5dfa0a67a8 2024-11-21T21:15:20,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:20,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223780198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:20,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:20,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:20,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223780203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:20,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:20,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:20,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223780204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:20,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:20,216 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/adbdbe7c41494e6f87a29c5dfa0a67a8, entries=150, sequenceid=33, filesize=11.7 K 2024-11-21T21:15:20,219 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 4a01a47e5e966934386d5c50a408380a in 153ms, sequenceid=33, compaction requested=false 2024-11-21T21:15:20,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:20,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:20,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-21T21:15:20,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-21T21:15:20,223 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-21T21:15:20,223 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2390 sec 2024-11-21T21:15:20,225 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.2480 sec 2024-11-21T21:15:20,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 4a01a47e5e966934386d5c50a408380a 2024-11-21T21:15:20,313 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a01a47e5e966934386d5c50a408380a 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-21T21:15:20,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=A 2024-11-21T21:15:20,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:20,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=B 2024-11-21T21:15:20,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:20,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=C 2024-11-21T21:15:20,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:20,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/98a10f6d966045a889d61a74cac91223 is 50, key is test_row_0/A:col10/1732223720313/Put/seqid=0 2024-11-21T21:15:20,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:20,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223780339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:20,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:20,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223780340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:20,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:20,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:20,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:20,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223780342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:20,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:20,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:20,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223780342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:20,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223780343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:20,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:20,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:20,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:20,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742067_1243 (size=12001) 2024-11-21T21:15:20,352 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/98a10f6d966045a889d61a74cac91223 2024-11-21T21:15:20,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/1c53bd25143b40bba76313bf6681a599 is 50, key is test_row_0/B:col10/1732223720313/Put/seqid=0 2024-11-21T21:15:20,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742068_1244 (size=12001) 2024-11-21T21:15:20,390 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/1c53bd25143b40bba76313bf6681a599 2024-11-21T21:15:20,409 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/f2409a7e5fd5477ab565ec6aef7ee4d1 is 50, key is test_row_0/C:col10/1732223720313/Put/seqid=0 2024-11-21T21:15:20,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742069_1245 (size=12001) 2024-11-21T21:15:20,439 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/f2409a7e5fd5477ab565ec6aef7ee4d1 2024-11-21T21:15:20,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:20,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223780451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:20,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:20,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223780451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:20,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:20,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:20,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/98a10f6d966045a889d61a74cac91223 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/98a10f6d966045a889d61a74cac91223 2024-11-21T21:15:20,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/98a10f6d966045a889d61a74cac91223, entries=150, sequenceid=49, filesize=11.7 K 2024-11-21T21:15:20,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/1c53bd25143b40bba76313bf6681a599 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/1c53bd25143b40bba76313bf6681a599 2024-11-21T21:15:20,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:20,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223780462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:20,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/1c53bd25143b40bba76313bf6681a599, entries=150, sequenceid=49, filesize=11.7 K 2024-11-21T21:15:20,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:20,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223780462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:20,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:20,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:20,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:20,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223780462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:20,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:20,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/f2409a7e5fd5477ab565ec6aef7ee4d1 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/f2409a7e5fd5477ab565ec6aef7ee4d1 2024-11-21T21:15:20,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/f2409a7e5fd5477ab565ec6aef7ee4d1, entries=150, sequenceid=49, filesize=11.7 K 2024-11-21T21:15:20,475 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 4a01a47e5e966934386d5c50a408380a in 162ms, sequenceid=49, compaction requested=true 2024-11-21T21:15:20,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:20,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a01a47e5e966934386d5c50a408380a:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:15:20,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:20,475 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:15:20,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a01a47e5e966934386d5c50a408380a:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:15:20,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:20,475 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:15:20,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a01a47e5e966934386d5c50a408380a:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:15:20,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:15:20,477 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:15:20,477 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:15:20,477 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): 4a01a47e5e966934386d5c50a408380a/A is initiating minor compaction (all files) 2024-11-21T21:15:20,477 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): 4a01a47e5e966934386d5c50a408380a/B is initiating minor compaction (all files) 2024-11-21T21:15:20,477 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a01a47e5e966934386d5c50a408380a/A in TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:20,477 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a01a47e5e966934386d5c50a408380a/B in TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:20,477 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/a455f76a627548e7b993e7e3547668d2, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/9048cd595602481bb4fe7f0922900b3b, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/98a10f6d966045a889d61a74cac91223] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp, totalSize=35.2 K 2024-11-21T21:15:20,477 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/9eecf6bf0d0b424ab50005316b35b16c, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/b445f33fb2b3411e90ad7684d7c7f2f2, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/1c53bd25143b40bba76313bf6681a599] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp, totalSize=35.2 K 2024-11-21T21:15:20,477 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting a455f76a627548e7b993e7e3547668d2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732223718986 2024-11-21T21:15:20,477 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 9eecf6bf0d0b424ab50005316b35b16c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732223718986 2024-11-21T21:15:20,478 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9048cd595602481bb4fe7f0922900b3b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732223719034 2024-11-21T21:15:20,479 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting b445f33fb2b3411e90ad7684d7c7f2f2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732223719034 2024-11-21T21:15:20,479 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98a10f6d966045a889d61a74cac91223, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732223720195 2024-11-21T21:15:20,479 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c53bd25143b40bba76313bf6681a599, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732223720195 2024-11-21T21:15:20,508 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a01a47e5e966934386d5c50a408380a#A#compaction#207 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:20,509 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/0db85a7d6bfe4e72a53f2e40e39f98bb is 50, key is test_row_0/A:col10/1732223720313/Put/seqid=0 2024-11-21T21:15:20,530 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a01a47e5e966934386d5c50a408380a#B#compaction#208 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:20,530 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/e529b5857bb14522b308609d2e5a7831 is 50, key is test_row_0/B:col10/1732223720313/Put/seqid=0 2024-11-21T21:15:20,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742070_1246 (size=12104) 2024-11-21T21:15:20,541 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/0db85a7d6bfe4e72a53f2e40e39f98bb as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/0db85a7d6bfe4e72a53f2e40e39f98bb 2024-11-21T21:15:20,547 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a01a47e5e966934386d5c50a408380a/A of 4a01a47e5e966934386d5c50a408380a into 0db85a7d6bfe4e72a53f2e40e39f98bb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:20,547 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:20,547 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., storeName=4a01a47e5e966934386d5c50a408380a/A, priority=13, startTime=1732223720475; duration=0sec 2024-11-21T21:15:20,547 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:15:20,547 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a01a47e5e966934386d5c50a408380a:A 2024-11-21T21:15:20,547 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:15:20,548 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:15:20,548 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): 4a01a47e5e966934386d5c50a408380a/C is initiating minor compaction (all files) 2024-11-21T21:15:20,549 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a01a47e5e966934386d5c50a408380a/C in TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:20,549 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/e512e73b5f35445eaddfcfb9e3ad1691, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/adbdbe7c41494e6f87a29c5dfa0a67a8, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/f2409a7e5fd5477ab565ec6aef7ee4d1] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp, totalSize=35.2 K 2024-11-21T21:15:20,549 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting e512e73b5f35445eaddfcfb9e3ad1691, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732223718986 2024-11-21T21:15:20,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742071_1247 (size=12104) 2024-11-21T21:15:20,550 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting adbdbe7c41494e6f87a29c5dfa0a67a8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732223719034 2024-11-21T21:15:20,551 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2409a7e5fd5477ab565ec6aef7ee4d1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732223720195 2024-11-21T21:15:20,556 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/e529b5857bb14522b308609d2e5a7831 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/e529b5857bb14522b308609d2e5a7831 2024-11-21T21:15:20,565 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a01a47e5e966934386d5c50a408380a#C#compaction#209 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:20,565 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/c0e15157cd344c6188bf5ad33d8d06f0 is 50, key is test_row_0/C:col10/1732223720313/Put/seqid=0 2024-11-21T21:15:20,568 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a01a47e5e966934386d5c50a408380a/B of 4a01a47e5e966934386d5c50a408380a into e529b5857bb14522b308609d2e5a7831(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:20,568 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:20,568 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., storeName=4a01a47e5e966934386d5c50a408380a/B, priority=13, startTime=1732223720475; duration=0sec 2024-11-21T21:15:20,568 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:20,568 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a01a47e5e966934386d5c50a408380a:B 2024-11-21T21:15:20,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742072_1248 (size=12104) 2024-11-21T21:15:20,592 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/c0e15157cd344c6188bf5ad33d8d06f0 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/c0e15157cd344c6188bf5ad33d8d06f0 2024-11-21T21:15:20,598 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a01a47e5e966934386d5c50a408380a/C of 4a01a47e5e966934386d5c50a408380a into c0e15157cd344c6188bf5ad33d8d06f0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:20,598 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:20,598 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., storeName=4a01a47e5e966934386d5c50a408380a/C, priority=13, startTime=1732223720475; duration=0sec 2024-11-21T21:15:20,598 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:20,598 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a01a47e5e966934386d5c50a408380a:C 2024-11-21T21:15:20,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 4a01a47e5e966934386d5c50a408380a 2024-11-21T21:15:20,663 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a01a47e5e966934386d5c50a408380a 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-21T21:15:20,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=A 2024-11-21T21:15:20,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:20,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=B 2024-11-21T21:15:20,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:20,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=C 2024-11-21T21:15:20,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:20,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/31e15b07410b456686c42453c98583a9 is 50, key is test_row_0/A:col10/1732223720661/Put/seqid=0 2024-11-21T21:15:20,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742073_1249 (size=12001) 2024-11-21T21:15:20,678 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/31e15b07410b456686c42453c98583a9 2024-11-21T21:15:20,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:20,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223780685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:20,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:20,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223780685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:20,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:20,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:20,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:20,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223780685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:20,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:20,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:20,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223780688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:20,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:20,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:20,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223780688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:20,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:20,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/a5806a032c1e4ef6ac931fcbdfd8fac7 is 50, key is test_row_0/B:col10/1732223720661/Put/seqid=0 2024-11-21T21:15:20,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742074_1250 (size=12001) 2024-11-21T21:15:20,709 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/a5806a032c1e4ef6ac931fcbdfd8fac7 2024-11-21T21:15:20,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/34214f256ddf49b29f7a8ac453f0155a is 50, key is test_row_0/C:col10/1732223720661/Put/seqid=0 2024-11-21T21:15:20,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742075_1251 (size=12001) 2024-11-21T21:15:20,728 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/34214f256ddf49b29f7a8ac453f0155a 2024-11-21T21:15:20,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/31e15b07410b456686c42453c98583a9 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/31e15b07410b456686c42453c98583a9 2024-11-21T21:15:20,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/31e15b07410b456686c42453c98583a9, entries=150, sequenceid=71, filesize=11.7 K 2024-11-21T21:15:20,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/a5806a032c1e4ef6ac931fcbdfd8fac7 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/a5806a032c1e4ef6ac931fcbdfd8fac7 2024-11-21T21:15:20,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/a5806a032c1e4ef6ac931fcbdfd8fac7, entries=150, sequenceid=71, filesize=11.7 K 2024-11-21T21:15:20,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/34214f256ddf49b29f7a8ac453f0155a as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/34214f256ddf49b29f7a8ac453f0155a 2024-11-21T21:15:20,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/34214f256ddf49b29f7a8ac453f0155a, entries=150, sequenceid=71, filesize=11.7 K 2024-11-21T21:15:20,752 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 4a01a47e5e966934386d5c50a408380a in 90ms, sequenceid=71, compaction requested=false 2024-11-21T21:15:20,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:20,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 4a01a47e5e966934386d5c50a408380a 2024-11-21T21:15:20,803 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a01a47e5e966934386d5c50a408380a 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-21T21:15:20,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=A 2024-11-21T21:15:20,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:20,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=B 2024-11-21T21:15:20,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:20,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=C 2024-11-21T21:15:20,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:20,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/a737dea74a4c48c38041c17276fa6eb0 is 50, key is test_row_0/A:col10/1732223720676/Put/seqid=0 2024-11-21T21:15:20,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742076_1252 (size=16681) 2024-11-21T21:15:20,824 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/a737dea74a4c48c38041c17276fa6eb0 2024-11-21T21:15:20,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/cc48533822ba4b719815a7b96fb96f11 is 50, key is test_row_0/B:col10/1732223720676/Put/seqid=0 2024-11-21T21:15:20,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:20,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223780829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:20,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:20,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:20,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223780833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:20,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:20,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:20,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223780833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:20,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:20,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742077_1253 (size=12001) 2024-11-21T21:15:20,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:20,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223780940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:20,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:20,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223780940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:20,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:20,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:20,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:20,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223780950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:20,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:20,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:21,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:21,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223781000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:21,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:21,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223781000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:21,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:21,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:21,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:21,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:21,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:21,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:21,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-21T21:15:21,140 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:15:21,141 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:15:21,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-21T21:15:21,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-21T21:15:21,144 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:15:21,145 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:15:21,145 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:15:21,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:21,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223781150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:21,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:21,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223781150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:21,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:21,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:21,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:21,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:21,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:21,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:21,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:21,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223781160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:21,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:21,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:21,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:21,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-21T21:15:21,255 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/cc48533822ba4b719815a7b96fb96f11 2024-11-21T21:15:21,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/835014565f2647178301bd67ab9dbe0a is 50, key is test_row_0/C:col10/1732223720676/Put/seqid=0 2024-11-21T21:15:21,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742078_1254 (size=12001) 2024-11-21T21:15:21,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-21T21:15:21,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:21,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:21,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:21,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:21,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:21,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:21,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-21T21:15:21,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:21,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:21,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:21,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:21,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:21,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:21,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-21T21:15:21,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:21,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223781460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:21,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:21,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223781460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:21,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:21,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:21,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:21,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:21,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:21,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:21,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:21,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223781471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:21,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:21,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:21,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:21,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:21,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223781511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:21,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:21,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:21,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:21,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:21,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223781511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:21,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:21,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:21,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:21,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-21T21:15:21,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:21,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:21,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:21,609 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:21,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:21,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:21,668 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/835014565f2647178301bd67ab9dbe0a 2024-11-21T21:15:21,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/a737dea74a4c48c38041c17276fa6eb0 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/a737dea74a4c48c38041c17276fa6eb0 2024-11-21T21:15:21,681 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/a737dea74a4c48c38041c17276fa6eb0, entries=250, sequenceid=87, filesize=16.3 K 2024-11-21T21:15:21,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/cc48533822ba4b719815a7b96fb96f11 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/cc48533822ba4b719815a7b96fb96f11 2024-11-21T21:15:21,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/cc48533822ba4b719815a7b96fb96f11, entries=150, sequenceid=87, filesize=11.7 K 2024-11-21T21:15:21,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/835014565f2647178301bd67ab9dbe0a as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/835014565f2647178301bd67ab9dbe0a 2024-11-21T21:15:21,714 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/835014565f2647178301bd67ab9dbe0a, entries=150, sequenceid=87, filesize=11.7 K 2024-11-21T21:15:21,715 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 4a01a47e5e966934386d5c50a408380a in 913ms, sequenceid=87, compaction requested=true 2024-11-21T21:15:21,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:21,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a01a47e5e966934386d5c50a408380a:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:15:21,716 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:15:21,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:21,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a01a47e5e966934386d5c50a408380a:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:15:21,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:15:21,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a01a47e5e966934386d5c50a408380a:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:15:21,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T21:15:21,716 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:15:21,717 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:15:21,717 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): 4a01a47e5e966934386d5c50a408380a/A is initiating minor compaction (all files) 2024-11-21T21:15:21,717 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a01a47e5e966934386d5c50a408380a/A in TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:21,717 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/0db85a7d6bfe4e72a53f2e40e39f98bb, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/31e15b07410b456686c42453c98583a9, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/a737dea74a4c48c38041c17276fa6eb0] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp, totalSize=39.8 K 2024-11-21T21:15:21,719 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:15:21,719 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): 4a01a47e5e966934386d5c50a408380a/B is initiating minor compaction (all files) 2024-11-21T21:15:21,719 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a01a47e5e966934386d5c50a408380a/B in TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:21,719 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/e529b5857bb14522b308609d2e5a7831, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/a5806a032c1e4ef6ac931fcbdfd8fac7, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/cc48533822ba4b719815a7b96fb96f11] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp, totalSize=35.3 K 2024-11-21T21:15:21,720 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0db85a7d6bfe4e72a53f2e40e39f98bb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732223720195 2024-11-21T21:15:21,720 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31e15b07410b456686c42453c98583a9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1732223720340 2024-11-21T21:15:21,720 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting e529b5857bb14522b308609d2e5a7831, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732223720195 2024-11-21T21:15:21,721 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting a737dea74a4c48c38041c17276fa6eb0, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1732223720676 2024-11-21T21:15:21,722 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting a5806a032c1e4ef6ac931fcbdfd8fac7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1732223720340 2024-11-21T21:15:21,723 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting cc48533822ba4b719815a7b96fb96f11, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1732223720676 2024-11-21T21:15:21,743 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a01a47e5e966934386d5c50a408380a#A#compaction#216 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:21,743 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/5f284319c337491ca3e8f69ddf82201d is 50, key is test_row_0/A:col10/1732223720676/Put/seqid=0 2024-11-21T21:15:21,746 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a01a47e5e966934386d5c50a408380a#B#compaction#217 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:21,747 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/62f69a709d26449ab44f4fe32057d1f8 is 50, key is test_row_0/B:col10/1732223720676/Put/seqid=0 2024-11-21T21:15:21,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-21T21:15:21,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:21,763 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 4a01a47e5e966934386d5c50a408380a 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-21T21:15:21,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=A 2024-11-21T21:15:21,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:21,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=B 2024-11-21T21:15:21,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:21,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=C 2024-11-21T21:15:21,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:21,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-21T21:15:21,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/3e2169db2e9e469ba85af447192951f3 is 50, key is test_row_0/A:col10/1732223720829/Put/seqid=0 2024-11-21T21:15:21,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742079_1255 (size=12207) 2024-11-21T21:15:21,798 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/5f284319c337491ca3e8f69ddf82201d as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/5f284319c337491ca3e8f69ddf82201d 2024-11-21T21:15:21,805 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a01a47e5e966934386d5c50a408380a/A of 4a01a47e5e966934386d5c50a408380a into 5f284319c337491ca3e8f69ddf82201d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:21,805 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:21,805 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., storeName=4a01a47e5e966934386d5c50a408380a/A, priority=13, startTime=1732223721716; duration=0sec 2024-11-21T21:15:21,805 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:15:21,805 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a01a47e5e966934386d5c50a408380a:A 2024-11-21T21:15:21,805 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:15:21,806 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:15:21,807 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): 4a01a47e5e966934386d5c50a408380a/C is initiating minor compaction (all files) 2024-11-21T21:15:21,807 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a01a47e5e966934386d5c50a408380a/C in TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:21,807 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/c0e15157cd344c6188bf5ad33d8d06f0, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/34214f256ddf49b29f7a8ac453f0155a, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/835014565f2647178301bd67ab9dbe0a] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp, totalSize=35.3 K 2024-11-21T21:15:21,807 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0e15157cd344c6188bf5ad33d8d06f0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732223720195 2024-11-21T21:15:21,808 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34214f256ddf49b29f7a8ac453f0155a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1732223720340 2024-11-21T21:15:21,808 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 835014565f2647178301bd67ab9dbe0a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1732223720676 2024-11-21T21:15:21,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742080_1256 (size=12207) 2024-11-21T21:15:21,823 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/62f69a709d26449ab44f4fe32057d1f8 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/62f69a709d26449ab44f4fe32057d1f8 2024-11-21T21:15:21,831 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a01a47e5e966934386d5c50a408380a/B of 4a01a47e5e966934386d5c50a408380a into 62f69a709d26449ab44f4fe32057d1f8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:21,831 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:21,832 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., storeName=4a01a47e5e966934386d5c50a408380a/B, priority=13, startTime=1732223721716; duration=0sec 2024-11-21T21:15:21,832 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:21,832 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a01a47e5e966934386d5c50a408380a:B 2024-11-21T21:15:21,836 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a01a47e5e966934386d5c50a408380a#C#compaction#219 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:21,837 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/9054e285b9764831bb977ce50f8fbadc is 50, key is test_row_0/C:col10/1732223720676/Put/seqid=0 2024-11-21T21:15:21,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742081_1257 (size=12001) 2024-11-21T21:15:21,838 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/3e2169db2e9e469ba85af447192951f3 2024-11-21T21:15:21,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/a2506d22634b43be9f4f11f3aa29b30d is 50, key is test_row_0/B:col10/1732223720829/Put/seqid=0 2024-11-21T21:15:21,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742082_1258 (size=12207) 2024-11-21T21:15:21,922 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/9054e285b9764831bb977ce50f8fbadc as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/9054e285b9764831bb977ce50f8fbadc 2024-11-21T21:15:21,930 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a01a47e5e966934386d5c50a408380a/C of 4a01a47e5e966934386d5c50a408380a into 9054e285b9764831bb977ce50f8fbadc(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:21,930 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:21,930 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., storeName=4a01a47e5e966934386d5c50a408380a/C, priority=13, startTime=1732223721716; duration=0sec 2024-11-21T21:15:21,930 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:21,930 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a01a47e5e966934386d5c50a408380a:C 2024-11-21T21:15:21,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742083_1259 (size=12001) 2024-11-21T21:15:21,958 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/a2506d22634b43be9f4f11f3aa29b30d 2024-11-21T21:15:21,979 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T21:15:21,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/dc2fbbbb5dec40ed83bfbb6d96477b20 is 50, key is test_row_0/C:col10/1732223720829/Put/seqid=0 2024-11-21T21:15:21,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:21,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 4a01a47e5e966934386d5c50a408380a 2024-11-21T21:15:22,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742084_1260 (size=12001) 2024-11-21T21:15:22,029 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/dc2fbbbb5dec40ed83bfbb6d96477b20 2024-11-21T21:15:22,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/3e2169db2e9e469ba85af447192951f3 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/3e2169db2e9e469ba85af447192951f3 2024-11-21T21:15:22,044 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/3e2169db2e9e469ba85af447192951f3, entries=150, sequenceid=103, filesize=11.7 K 2024-11-21T21:15:22,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/a2506d22634b43be9f4f11f3aa29b30d as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/a2506d22634b43be9f4f11f3aa29b30d 2024-11-21T21:15:22,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:22,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223782034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:22,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:22,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:22,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:22,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:22,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223782048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:22,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:22,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:22,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:22,055 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/a2506d22634b43be9f4f11f3aa29b30d, entries=150, sequenceid=103, filesize=11.7 K 2024-11-21T21:15:22,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:22,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223782045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:22,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:22,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:22,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:22,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/dc2fbbbb5dec40ed83bfbb6d96477b20 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/dc2fbbbb5dec40ed83bfbb6d96477b20 2024-11-21T21:15:22,062 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/dc2fbbbb5dec40ed83bfbb6d96477b20, entries=150, sequenceid=103, filesize=11.7 K 2024-11-21T21:15:22,063 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 4a01a47e5e966934386d5c50a408380a in 300ms, sequenceid=103, compaction requested=false 2024-11-21T21:15:22,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:22,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:22,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-21T21:15:22,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-21T21:15:22,068 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-21T21:15:22,068 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 920 msec 2024-11-21T21:15:22,071 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 929 msec 2024-11-21T21:15:22,167 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a01a47e5e966934386d5c50a408380a 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-21T21:15:22,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=A 2024-11-21T21:15:22,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:22,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=B 2024-11-21T21:15:22,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:22,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=C 2024-11-21T21:15:22,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:22,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 4a01a47e5e966934386d5c50a408380a 2024-11-21T21:15:22,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/ebf39cb4c7da43c3b0af0ad5a2737c19 is 50, key is test_row_0/A:col10/1732223722164/Put/seqid=0 2024-11-21T21:15:22,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:22,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:22,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223782203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:22,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:22,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:22,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:22,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223782198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:22,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:22,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:22,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:22,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:22,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223782209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:22,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:22,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:22,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:22,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742085_1261 (size=14341) 2024-11-21T21:15:22,226 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/ebf39cb4c7da43c3b0af0ad5a2737c19 2024-11-21T21:15:22,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/5568f8c6f67e47058878abc7e00eb217 is 50, key is test_row_0/B:col10/1732223722164/Put/seqid=0 2024-11-21T21:15:22,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-21T21:15:22,280 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:15:22,283 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:15:22,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-21T21:15:22,286 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:15:22,287 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:15:22,287 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:15:22,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-21T21:15:22,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742086_1262 (size=12001) 2024-11-21T21:15:22,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/5568f8c6f67e47058878abc7e00eb217 2024-11-21T21:15:22,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/d8a67e21e6164c8387426b2c4c11314b is 50, key is test_row_0/C:col10/1732223722164/Put/seqid=0 2024-11-21T21:15:22,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:22,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223782320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:22,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:22,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223782320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:22,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:22,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:22,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:22,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:22,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223782320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:22,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:22,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:22,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:22,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:22,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:22,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:22,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742087_1263 (size=12001) 2024-11-21T21:15:22,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/d8a67e21e6164c8387426b2c4c11314b 2024-11-21T21:15:22,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/ebf39cb4c7da43c3b0af0ad5a2737c19 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/ebf39cb4c7da43c3b0af0ad5a2737c19 2024-11-21T21:15:22,360 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/ebf39cb4c7da43c3b0af0ad5a2737c19, entries=200, sequenceid=123, filesize=14.0 K 2024-11-21T21:15:22,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/5568f8c6f67e47058878abc7e00eb217 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/5568f8c6f67e47058878abc7e00eb217 2024-11-21T21:15:22,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/5568f8c6f67e47058878abc7e00eb217, entries=150, sequenceid=123, filesize=11.7 K 2024-11-21T21:15:22,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/d8a67e21e6164c8387426b2c4c11314b as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/d8a67e21e6164c8387426b2c4c11314b 2024-11-21T21:15:22,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/d8a67e21e6164c8387426b2c4c11314b, entries=150, sequenceid=123, filesize=11.7 K 2024-11-21T21:15:22,388 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 4a01a47e5e966934386d5c50a408380a in 221ms, sequenceid=123, compaction requested=true 2024-11-21T21:15:22,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:22,388 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:15:22,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a01a47e5e966934386d5c50a408380a:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:15:22,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:22,389 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:15:22,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-21T21:15:22,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a01a47e5e966934386d5c50a408380a:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:15:22,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:22,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a01a47e5e966934386d5c50a408380a:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:15:22,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:15:22,391 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:15:22,391 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): 4a01a47e5e966934386d5c50a408380a/A is initiating minor compaction (all files) 2024-11-21T21:15:22,391 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a01a47e5e966934386d5c50a408380a/A in TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:22,391 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/5f284319c337491ca3e8f69ddf82201d, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/3e2169db2e9e469ba85af447192951f3, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/ebf39cb4c7da43c3b0af0ad5a2737c19] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp, totalSize=37.6 K 2024-11-21T21:15:22,392 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:15:22,392 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): 4a01a47e5e966934386d5c50a408380a/B is initiating minor compaction (all files) 2024-11-21T21:15:22,392 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f284319c337491ca3e8f69ddf82201d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1732223720676 2024-11-21T21:15:22,392 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a01a47e5e966934386d5c50a408380a/B in TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:22,392 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/62f69a709d26449ab44f4fe32057d1f8, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/a2506d22634b43be9f4f11f3aa29b30d, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/5568f8c6f67e47058878abc7e00eb217] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp, totalSize=35.4 K 2024-11-21T21:15:22,392 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e2169db2e9e469ba85af447192951f3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732223720827 2024-11-21T21:15:22,393 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebf39cb4c7da43c3b0af0ad5a2737c19, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1732223722023 2024-11-21T21:15:22,393 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 62f69a709d26449ab44f4fe32057d1f8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1732223720676 2024-11-21T21:15:22,393 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting a2506d22634b43be9f4f11f3aa29b30d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732223720827 2024-11-21T21:15:22,394 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 5568f8c6f67e47058878abc7e00eb217, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1732223722031 2024-11-21T21:15:22,419 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a01a47e5e966934386d5c50a408380a#B#compaction#225 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:22,419 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/d4537c9d37734294817c9f66cfff6df4 is 50, key is test_row_0/B:col10/1732223722164/Put/seqid=0 2024-11-21T21:15:22,439 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a01a47e5e966934386d5c50a408380a#A#compaction#226 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:22,439 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/9a11bdb8b14e40c397f072bb9cccf302 is 50, key is test_row_0/A:col10/1732223722164/Put/seqid=0 2024-11-21T21:15:22,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T21:15:22,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:22,441 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 4a01a47e5e966934386d5c50a408380a 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-21T21:15:22,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=A 2024-11-21T21:15:22,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:22,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=B 2024-11-21T21:15:22,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:22,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=C 2024-11-21T21:15:22,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:22,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/3d0f1973382f4006829bca962612fd82 is 50, key is test_row_0/A:col10/1732223722204/Put/seqid=0 2024-11-21T21:15:22,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742088_1264 (size=12309) 2024-11-21T21:15:22,481 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/d4537c9d37734294817c9f66cfff6df4 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/d4537c9d37734294817c9f66cfff6df4 2024-11-21T21:15:22,487 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a01a47e5e966934386d5c50a408380a/B of 4a01a47e5e966934386d5c50a408380a into d4537c9d37734294817c9f66cfff6df4(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:22,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742089_1265 (size=12309) 2024-11-21T21:15:22,489 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:22,489 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., storeName=4a01a47e5e966934386d5c50a408380a/B, priority=13, startTime=1732223722389; duration=0sec 2024-11-21T21:15:22,489 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:15:22,489 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a01a47e5e966934386d5c50a408380a:B 2024-11-21T21:15:22,489 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:15:22,491 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:15:22,491 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): 4a01a47e5e966934386d5c50a408380a/C is initiating minor compaction (all files) 2024-11-21T21:15:22,492 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a01a47e5e966934386d5c50a408380a/C in TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:22,492 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/9054e285b9764831bb977ce50f8fbadc, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/dc2fbbbb5dec40ed83bfbb6d96477b20, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/d8a67e21e6164c8387426b2c4c11314b] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp, totalSize=35.4 K 2024-11-21T21:15:22,493 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 9054e285b9764831bb977ce50f8fbadc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1732223720676 2024-11-21T21:15:22,493 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting dc2fbbbb5dec40ed83bfbb6d96477b20, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732223720827 2024-11-21T21:15:22,494 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting d8a67e21e6164c8387426b2c4c11314b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1732223722031 2024-11-21T21:15:22,496 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/9a11bdb8b14e40c397f072bb9cccf302 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/9a11bdb8b14e40c397f072bb9cccf302 2024-11-21T21:15:22,509 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a01a47e5e966934386d5c50a408380a/A of 4a01a47e5e966934386d5c50a408380a into 9a11bdb8b14e40c397f072bb9cccf302(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:22,509 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:22,509 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., storeName=4a01a47e5e966934386d5c50a408380a/A, priority=13, startTime=1732223722388; duration=0sec 2024-11-21T21:15:22,509 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:22,509 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a01a47e5e966934386d5c50a408380a:A 2024-11-21T21:15:22,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742090_1266 (size=12151) 2024-11-21T21:15:22,517 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a01a47e5e966934386d5c50a408380a#C#compaction#228 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:22,518 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/3d0f1973382f4006829bca962612fd82 2024-11-21T21:15:22,518 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/ab317961abc74c87a714dd75e56f47ce is 50, key is test_row_0/C:col10/1732223722164/Put/seqid=0 2024-11-21T21:15:22,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 4a01a47e5e966934386d5c50a408380a 2024-11-21T21:15:22,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:22,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223782551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:22,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:22,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:22,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:22,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:22,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742091_1267 (size=12309) 2024-11-21T21:15:22,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:22,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223782552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:22,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:22,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223782552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:22,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/fe5cda48c84f4303ba75bd4426961040 is 50, key is test_row_0/B:col10/1732223722204/Put/seqid=0 2024-11-21T21:15:22,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:22,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:22,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:22,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:22,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:22,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:22,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:22,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223782557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:22,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:22,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:22,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223782551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:22,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:22,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:22,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:22,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742092_1268 (size=12151) 2024-11-21T21:15:22,599 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/fe5cda48c84f4303ba75bd4426961040 2024-11-21T21:15:22,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-21T21:15:22,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/cc6b85b8725e4ce2af48e49d51a17907 is 50, key is test_row_0/C:col10/1732223722204/Put/seqid=0 2024-11-21T21:15:22,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742093_1269 (size=12151) 2024-11-21T21:15:22,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:22,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223782661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:22,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:22,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:22,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:22,669 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/cc6b85b8725e4ce2af48e49d51a17907 2024-11-21T21:15:22,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/3d0f1973382f4006829bca962612fd82 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/3d0f1973382f4006829bca962612fd82 2024-11-21T21:15:22,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:22,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:22,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223782671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:22,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223782671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:22,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:22,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:22,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:22,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:22,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223782672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:22,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:22,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:22,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:22,683 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/3d0f1973382f4006829bca962612fd82, entries=150, sequenceid=138, filesize=11.9 K 2024-11-21T21:15:22,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/fe5cda48c84f4303ba75bd4426961040 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/fe5cda48c84f4303ba75bd4426961040 2024-11-21T21:15:22,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:22,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223782681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:22,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:22,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:22,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:22,690 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/fe5cda48c84f4303ba75bd4426961040, entries=150, sequenceid=138, filesize=11.9 K 2024-11-21T21:15:22,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/cc6b85b8725e4ce2af48e49d51a17907 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/cc6b85b8725e4ce2af48e49d51a17907 2024-11-21T21:15:22,697 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/cc6b85b8725e4ce2af48e49d51a17907, entries=150, sequenceid=138, filesize=11.9 K 2024-11-21T21:15:22,700 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 4a01a47e5e966934386d5c50a408380a in 260ms, sequenceid=138, compaction requested=false 2024-11-21T21:15:22,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:22,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:22,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-21T21:15:22,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:22,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:22,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-21T21:15:22,704 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-21T21:15:22,704 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 415 msec 2024-11-21T21:15:22,706 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 422 msec 2024-11-21T21:15:22,879 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a01a47e5e966934386d5c50a408380a 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-21T21:15:22,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=A 2024-11-21T21:15:22,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:22,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=B 2024-11-21T21:15:22,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:22,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=C 2024-11-21T21:15:22,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:22,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 4a01a47e5e966934386d5c50a408380a 2024-11-21T21:15:22,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/8cd894104d1a4dc0bb781a95a4fcc784 is 50, key is test_row_0/A:col10/1732223722555/Put/seqid=0 2024-11-21T21:15:22,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-21T21:15:22,911 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:15:22,911 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:15:22,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-21T21:15:22,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T21:15:22,914 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:15:22,915 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:15:22,915 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:15:22,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:22,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223782912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:22,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:22,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:22,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:22,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223782913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:22,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:22,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:22,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:22,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742094_1270 (size=14541) 2024-11-21T21:15:22,926 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/8cd894104d1a4dc0bb781a95a4fcc784 2024-11-21T21:15:22,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223782921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:22,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:22,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:22,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223782923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:22,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:22,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:22,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:22,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:22,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223782925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:22,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:22,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:22,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:22,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/ae0146d508bc4f6fa0c54ca62a031d0f is 50, key is test_row_0/B:col10/1732223722555/Put/seqid=0 2024-11-21T21:15:22,967 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/ab317961abc74c87a714dd75e56f47ce as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/ab317961abc74c87a714dd75e56f47ce 2024-11-21T21:15:22,975 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a01a47e5e966934386d5c50a408380a/C of 4a01a47e5e966934386d5c50a408380a into ab317961abc74c87a714dd75e56f47ce(size=12.0 K), total size for store is 23.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:22,975 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:22,975 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., storeName=4a01a47e5e966934386d5c50a408380a/C, priority=13, startTime=1732223722391; duration=0sec 2024-11-21T21:15:22,975 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:22,975 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a01a47e5e966934386d5c50a408380a:C 2024-11-21T21:15:22,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742095_1271 (size=12151) 2024-11-21T21:15:22,991 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/ae0146d508bc4f6fa0c54ca62a031d0f 2024-11-21T21:15:23,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/7e18558d45e947f2bbf657676bd833c5 is 50, key is test_row_0/C:col10/1732223722555/Put/seqid=0 2024-11-21T21:15:23,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T21:15:23,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:23,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223783031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:23,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:23,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:23,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:23,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:23,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223783034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:23,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:23,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:23,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:23,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:23,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223783040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:23,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:23,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:23,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:23,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:23,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223783040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:23,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:23,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:23,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:23,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:23,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223783040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:23,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:23,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:23,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:23,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T21:15:23,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:23,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:23,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:23,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:23,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:23,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:23,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742096_1272 (size=12151) 2024-11-21T21:15:23,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T21:15:23,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:23,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:23,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:23,222 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:23,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:23,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:23,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T21:15:23,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:23,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223783240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:23,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:23,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:23,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:23,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:23,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223783250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:23,255 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:23,255 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:23,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:23,255 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:23,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223783250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:23,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:23,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:23,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:23,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:23,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223783260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:23,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:23,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:23,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:23,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:23,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223783261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:23,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:23,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:23,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:23,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T21:15:23,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:23,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:23,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:23,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:23,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:23,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:23,471 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/7e18558d45e947f2bbf657676bd833c5 2024-11-21T21:15:23,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/8cd894104d1a4dc0bb781a95a4fcc784 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/8cd894104d1a4dc0bb781a95a4fcc784 2024-11-21T21:15:23,479 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/8cd894104d1a4dc0bb781a95a4fcc784, entries=200, sequenceid=158, filesize=14.2 K 2024-11-21T21:15:23,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/ae0146d508bc4f6fa0c54ca62a031d0f as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/ae0146d508bc4f6fa0c54ca62a031d0f 2024-11-21T21:15:23,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/ae0146d508bc4f6fa0c54ca62a031d0f, entries=150, sequenceid=158, filesize=11.9 K 2024-11-21T21:15:23,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/7e18558d45e947f2bbf657676bd833c5 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/7e18558d45e947f2bbf657676bd833c5 2024-11-21T21:15:23,496 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/7e18558d45e947f2bbf657676bd833c5, entries=150, sequenceid=158, filesize=11.9 K 2024-11-21T21:15:23,497 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 4a01a47e5e966934386d5c50a408380a in 618ms, sequenceid=158, compaction requested=true 2024-11-21T21:15:23,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:23,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a01a47e5e966934386d5c50a408380a:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:15:23,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:23,497 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:15:23,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a01a47e5e966934386d5c50a408380a:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:15:23,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:23,497 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:15:23,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a01a47e5e966934386d5c50a408380a:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:15:23,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:15:23,500 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39001 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:15:23,500 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:15:23,500 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): 4a01a47e5e966934386d5c50a408380a/A is initiating minor compaction (all files) 2024-11-21T21:15:23,500 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): 4a01a47e5e966934386d5c50a408380a/B is initiating minor compaction (all files) 2024-11-21T21:15:23,500 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a01a47e5e966934386d5c50a408380a/B in TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:23,500 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a01a47e5e966934386d5c50a408380a/A in TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:23,500 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/9a11bdb8b14e40c397f072bb9cccf302, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/3d0f1973382f4006829bca962612fd82, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/8cd894104d1a4dc0bb781a95a4fcc784] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp, totalSize=38.1 K 2024-11-21T21:15:23,500 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/d4537c9d37734294817c9f66cfff6df4, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/fe5cda48c84f4303ba75bd4426961040, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/ae0146d508bc4f6fa0c54ca62a031d0f] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp, totalSize=35.8 K 2024-11-21T21:15:23,500 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a11bdb8b14e40c397f072bb9cccf302, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1732223722031 2024-11-21T21:15:23,501 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting d4537c9d37734294817c9f66cfff6df4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1732223722031 2024-11-21T21:15:23,501 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d0f1973382f4006829bca962612fd82, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1732223722196 2024-11-21T21:15:23,501 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting fe5cda48c84f4303ba75bd4426961040, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1732223722196 2024-11-21T21:15:23,501 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8cd894104d1a4dc0bb781a95a4fcc784, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732223722543 2024-11-21T21:15:23,503 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting ae0146d508bc4f6fa0c54ca62a031d0f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732223722543 2024-11-21T21:15:23,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T21:15:23,531 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a01a47e5e966934386d5c50a408380a#B#compaction#234 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:23,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:23,531 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 4a01a47e5e966934386d5c50a408380a 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-21T21:15:23,531 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/f17b05b8897a45b38d88f45bff84d08d is 50, key is test_row_0/B:col10/1732223722555/Put/seqid=0 2024-11-21T21:15:23,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=A 2024-11-21T21:15:23,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:23,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=B 2024-11-21T21:15:23,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:23,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=C 2024-11-21T21:15:23,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:23,538 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a01a47e5e966934386d5c50a408380a#A#compaction#235 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:23,539 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/a4f35c4f18e8435d82c6dc4fd5068a7c is 50, key is test_row_0/A:col10/1732223722555/Put/seqid=0 2024-11-21T21:15:23,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T21:15:23,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/2c45a49a279c4fb4987594bb98f021dd is 50, key is test_row_0/A:col10/1732223722922/Put/seqid=0 2024-11-21T21:15:23,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 4a01a47e5e966934386d5c50a408380a 2024-11-21T21:15:23,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:23,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742099_1275 (size=12151) 2024-11-21T21:15:23,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742098_1274 (size=12561) 2024-11-21T21:15:23,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742097_1273 (size=12561) 2024-11-21T21:15:23,588 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/f17b05b8897a45b38d88f45bff84d08d as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/f17b05b8897a45b38d88f45bff84d08d 2024-11-21T21:15:23,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:23,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223783588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:23,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:23,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:23,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:23,596 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a01a47e5e966934386d5c50a408380a/B of 4a01a47e5e966934386d5c50a408380a into f17b05b8897a45b38d88f45bff84d08d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:23,596 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:23,596 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., storeName=4a01a47e5e966934386d5c50a408380a/B, priority=13, startTime=1732223723497; duration=0sec 2024-11-21T21:15:23,596 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:15:23,596 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a01a47e5e966934386d5c50a408380a:B 2024-11-21T21:15:23,596 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:15:23,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:23,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223783591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:23,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:23,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:23,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:23,598 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:15:23,598 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): 4a01a47e5e966934386d5c50a408380a/C is initiating minor compaction (all files) 2024-11-21T21:15:23,598 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a01a47e5e966934386d5c50a408380a/C in TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:23,598 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/ab317961abc74c87a714dd75e56f47ce, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/cc6b85b8725e4ce2af48e49d51a17907, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/7e18558d45e947f2bbf657676bd833c5] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp, totalSize=35.8 K 2024-11-21T21:15:23,599 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting ab317961abc74c87a714dd75e56f47ce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1732223722031 2024-11-21T21:15:23,599 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting cc6b85b8725e4ce2af48e49d51a17907, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1732223722196 2024-11-21T21:15:23,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:23,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223783592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:23,599 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e18558d45e947f2bbf657676bd833c5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732223722543 2024-11-21T21:15:23,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:23,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:23,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223783593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:23,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223783594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:23,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:23,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:23,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:23,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:23,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:23,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:23,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:23,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:23,601 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:23,608 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a01a47e5e966934386d5c50a408380a#C#compaction#237 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:23,609 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/bce5506fac424df68ff76a881d6eebb9 is 50, key is test_row_0/C:col10/1732223722555/Put/seqid=0 2024-11-21T21:15:23,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742100_1276 (size=12561) 2024-11-21T21:15:23,627 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/bce5506fac424df68ff76a881d6eebb9 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/bce5506fac424df68ff76a881d6eebb9 2024-11-21T21:15:23,636 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a01a47e5e966934386d5c50a408380a/C of 4a01a47e5e966934386d5c50a408380a into bce5506fac424df68ff76a881d6eebb9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:23,636 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:23,636 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., storeName=4a01a47e5e966934386d5c50a408380a/C, priority=13, startTime=1732223723497; duration=0sec 2024-11-21T21:15:23,636 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:23,636 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a01a47e5e966934386d5c50a408380a:C 2024-11-21T21:15:23,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:23,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223783700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:23,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:23,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:23,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:23,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:23,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223783700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:23,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:23,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:23,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:23,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:23,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223783710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:23,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:23,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:23,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223783711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:23,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:23,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:23,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:23,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:23,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:23,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:23,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223783711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:23,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:23,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:23,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:23,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:23,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223783920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:23,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:23,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223783920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:23,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:23,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:23,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:23,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:23,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223783920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:23,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:23,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:23,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:23,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:23,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:23,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:23,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:23,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223783921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:23,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:23,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:23,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:23,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:23,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223783921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:23,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:23,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:23,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:23,969 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/2c45a49a279c4fb4987594bb98f021dd 2024-11-21T21:15:23,988 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/a4f35c4f18e8435d82c6dc4fd5068a7c as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/a4f35c4f18e8435d82c6dc4fd5068a7c 2024-11-21T21:15:23,998 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a01a47e5e966934386d5c50a408380a/A of 4a01a47e5e966934386d5c50a408380a into a4f35c4f18e8435d82c6dc4fd5068a7c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:23,998 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:23,998 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., storeName=4a01a47e5e966934386d5c50a408380a/A, priority=13, startTime=1732223723497; duration=0sec 2024-11-21T21:15:23,998 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:23,998 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a01a47e5e966934386d5c50a408380a:A 2024-11-21T21:15:24,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/ae0e9f578ee54753b8490818a0e969e4 is 50, key is test_row_0/B:col10/1732223722922/Put/seqid=0 2024-11-21T21:15:24,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T21:15:24,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742101_1277 (size=12151) 2024-11-21T21:15:24,074 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/ae0e9f578ee54753b8490818a0e969e4 2024-11-21T21:15:24,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/5dbaeb3eff5348abb6b8e89370a2208e is 50, key is test_row_0/C:col10/1732223722922/Put/seqid=0 2024-11-21T21:15:24,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742102_1278 (size=12151) 2024-11-21T21:15:24,105 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/5dbaeb3eff5348abb6b8e89370a2208e 2024-11-21T21:15:24,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/2c45a49a279c4fb4987594bb98f021dd as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/2c45a49a279c4fb4987594bb98f021dd 2024-11-21T21:15:24,122 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/2c45a49a279c4fb4987594bb98f021dd, entries=150, sequenceid=174, filesize=11.9 K 2024-11-21T21:15:24,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/ae0e9f578ee54753b8490818a0e969e4 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/ae0e9f578ee54753b8490818a0e969e4 2024-11-21T21:15:24,132 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/ae0e9f578ee54753b8490818a0e969e4, entries=150, sequenceid=174, filesize=11.9 K 2024-11-21T21:15:24,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/5dbaeb3eff5348abb6b8e89370a2208e as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/5dbaeb3eff5348abb6b8e89370a2208e 2024-11-21T21:15:24,143 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/5dbaeb3eff5348abb6b8e89370a2208e, entries=150, sequenceid=174, filesize=11.9 K 2024-11-21T21:15:24,145 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 4a01a47e5e966934386d5c50a408380a in 614ms, sequenceid=174, compaction requested=false 2024-11-21T21:15:24,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:24,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:24,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-21T21:15:24,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-21T21:15:24,150 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-21T21:15:24,150 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2320 sec 2024-11-21T21:15:24,152 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.2400 sec 2024-11-21T21:15:24,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 4a01a47e5e966934386d5c50a408380a 2024-11-21T21:15:24,234 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a01a47e5e966934386d5c50a408380a 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-21T21:15:24,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=A 2024-11-21T21:15:24,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:24,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=B 2024-11-21T21:15:24,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:24,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=C 2024-11-21T21:15:24,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:24,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/b78c7a36d44844399c9ed6bfd9543ab8 is 50, key is test_row_0/A:col10/1732223724232/Put/seqid=0 2024-11-21T21:15:24,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742103_1279 (size=12151) 2024-11-21T21:15:24,253 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/b78c7a36d44844399c9ed6bfd9543ab8 2024-11-21T21:15:24,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:24,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223784248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:24,255 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:24,255 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:24,255 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:24,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:24,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:24,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223784252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:24,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223784252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:24,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:24,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223784254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:24,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:24,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:24,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:24,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:24,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:24,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:24,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:24,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223784257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:24,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:24,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:24,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:24,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:24,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:24,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:24,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/dfe3e676bbb64e158ec9866b8696efe6 is 50, key is test_row_0/B:col10/1732223724232/Put/seqid=0 2024-11-21T21:15:24,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742104_1280 (size=12151) 2024-11-21T21:15:24,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:24,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223784361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:24,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:24,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:24,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:24,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:24,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223784370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:24,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:24,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:24,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:24,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:24,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223784370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:24,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:24,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223784370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:24,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:24,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:24,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:24,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:24,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223784370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:24,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:24,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:24,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:24,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:24,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:24,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:24,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:24,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223784570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:24,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:24,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:24,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:24,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:24,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223784580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:24,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:24,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:24,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:24,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:24,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223784580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:24,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:24,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:24,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:24,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:24,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223784580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:24,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:24,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:24,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:24,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:24,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223784590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:24,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:24,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:24,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:24,694 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/dfe3e676bbb64e158ec9866b8696efe6 2024-11-21T21:15:24,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/7a9d28bc952b4d10ad6095a7593b9302 is 50, key is test_row_0/C:col10/1732223724232/Put/seqid=0 2024-11-21T21:15:24,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742105_1281 (size=12151) 2024-11-21T21:15:24,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:24,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223784881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:24,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:24,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:24,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:24,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:24,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223784890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:24,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:24,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:24,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:24,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223784890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:24,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:24,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:24,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223784890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:24,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:24,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:24,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:24,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:24,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:24,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:24,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:24,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223784900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:24,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:24,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:24,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:25,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T21:15:25,060 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:15:25,061 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:15:25,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-21T21:15:25,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T21:15:25,066 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:15:25,068 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:15:25,068 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:15:25,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/7a9d28bc952b4d10ad6095a7593b9302 2024-11-21T21:15:25,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/b78c7a36d44844399c9ed6bfd9543ab8 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/b78c7a36d44844399c9ed6bfd9543ab8 2024-11-21T21:15:25,126 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/b78c7a36d44844399c9ed6bfd9543ab8, entries=150, sequenceid=195, filesize=11.9 K 2024-11-21T21:15:25,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/dfe3e676bbb64e158ec9866b8696efe6 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/dfe3e676bbb64e158ec9866b8696efe6 2024-11-21T21:15:25,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/dfe3e676bbb64e158ec9866b8696efe6, entries=150, sequenceid=195, filesize=11.9 K 2024-11-21T21:15:25,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/7a9d28bc952b4d10ad6095a7593b9302 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/7a9d28bc952b4d10ad6095a7593b9302 2024-11-21T21:15:25,145 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/7a9d28bc952b4d10ad6095a7593b9302, entries=150, sequenceid=195, filesize=11.9 K 2024-11-21T21:15:25,146 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 4a01a47e5e966934386d5c50a408380a in 912ms, sequenceid=195, compaction requested=true 2024-11-21T21:15:25,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:25,146 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:15:25,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a01a47e5e966934386d5c50a408380a:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:15:25,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:25,147 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:15:25,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a01a47e5e966934386d5c50a408380a:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:15:25,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:25,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a01a47e5e966934386d5c50a408380a:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:15:25,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:15:25,148 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:15:25,148 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): 4a01a47e5e966934386d5c50a408380a/A is initiating minor compaction (all files) 2024-11-21T21:15:25,148 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a01a47e5e966934386d5c50a408380a/A in TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:25,148 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/a4f35c4f18e8435d82c6dc4fd5068a7c, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/2c45a49a279c4fb4987594bb98f021dd, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/b78c7a36d44844399c9ed6bfd9543ab8] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp, totalSize=36.0 K 2024-11-21T21:15:25,148 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4f35c4f18e8435d82c6dc4fd5068a7c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732223722543 2024-11-21T21:15:25,149 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:15:25,149 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): 4a01a47e5e966934386d5c50a408380a/B is initiating minor compaction (all files) 2024-11-21T21:15:25,149 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a01a47e5e966934386d5c50a408380a/B in TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:25,149 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/f17b05b8897a45b38d88f45bff84d08d, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/ae0e9f578ee54753b8490818a0e969e4, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/dfe3e676bbb64e158ec9866b8696efe6] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp, totalSize=36.0 K 2024-11-21T21:15:25,149 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c45a49a279c4fb4987594bb98f021dd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732223722894 2024-11-21T21:15:25,149 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting f17b05b8897a45b38d88f45bff84d08d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732223722543 2024-11-21T21:15:25,150 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting ae0e9f578ee54753b8490818a0e969e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732223722894 2024-11-21T21:15:25,150 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting b78c7a36d44844399c9ed6bfd9543ab8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732223723591 2024-11-21T21:15:25,151 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting dfe3e676bbb64e158ec9866b8696efe6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732223723591 2024-11-21T21:15:25,163 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a01a47e5e966934386d5c50a408380a#A#compaction#243 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:25,164 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/d661c5c8fb954138ba84e4f96f12a887 is 50, key is test_row_0/A:col10/1732223724232/Put/seqid=0 2024-11-21T21:15:25,169 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a01a47e5e966934386d5c50a408380a#B#compaction#244 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:25,170 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/6b28d618aa2b4c349314ab6a45b913fd is 50, key is test_row_0/B:col10/1732223724232/Put/seqid=0 2024-11-21T21:15:25,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T21:15:25,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742106_1282 (size=12663) 2024-11-21T21:15:25,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742107_1283 (size=12663) 2024-11-21T21:15:25,202 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/6b28d618aa2b4c349314ab6a45b913fd as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/6b28d618aa2b4c349314ab6a45b913fd 2024-11-21T21:15:25,209 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a01a47e5e966934386d5c50a408380a/B of 4a01a47e5e966934386d5c50a408380a into 6b28d618aa2b4c349314ab6a45b913fd(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:25,209 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:25,209 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., storeName=4a01a47e5e966934386d5c50a408380a/B, priority=13, startTime=1732223725147; duration=0sec 2024-11-21T21:15:25,209 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:15:25,209 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a01a47e5e966934386d5c50a408380a:B 2024-11-21T21:15:25,209 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:15:25,217 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:15:25,217 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): 4a01a47e5e966934386d5c50a408380a/C is initiating minor compaction (all files) 2024-11-21T21:15:25,217 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a01a47e5e966934386d5c50a408380a/C in TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:25,217 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/bce5506fac424df68ff76a881d6eebb9, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/5dbaeb3eff5348abb6b8e89370a2208e, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/7a9d28bc952b4d10ad6095a7593b9302] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp, totalSize=36.0 K 2024-11-21T21:15:25,218 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting bce5506fac424df68ff76a881d6eebb9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732223722543 2024-11-21T21:15:25,218 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 5dbaeb3eff5348abb6b8e89370a2208e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732223722894 2024-11-21T21:15:25,218 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a9d28bc952b4d10ad6095a7593b9302, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732223723591 2024-11-21T21:15:25,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-21T21:15:25,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:25,221 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 4a01a47e5e966934386d5c50a408380a 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-21T21:15:25,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=A 2024-11-21T21:15:25,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:25,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=B 2024-11-21T21:15:25,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:25,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=C 2024-11-21T21:15:25,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:25,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/b727995f8a7f46f68d7834f534373a6e is 50, key is test_row_0/A:col10/1732223724248/Put/seqid=0 2024-11-21T21:15:25,234 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a01a47e5e966934386d5c50a408380a#C#compaction#246 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:25,235 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/1185a922c7ee43b4bf5795ce00cd83c3 is 50, key is test_row_0/C:col10/1732223724232/Put/seqid=0 2024-11-21T21:15:25,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742108_1284 (size=12151) 2024-11-21T21:15:25,273 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/b727995f8a7f46f68d7834f534373a6e 2024-11-21T21:15:25,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742109_1285 (size=12663) 2024-11-21T21:15:25,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/6c4cb720c43b424f8de2e550f9141f53 is 50, key is test_row_0/B:col10/1732223724248/Put/seqid=0 2024-11-21T21:15:25,290 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/1185a922c7ee43b4bf5795ce00cd83c3 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/1185a922c7ee43b4bf5795ce00cd83c3 2024-11-21T21:15:25,296 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a01a47e5e966934386d5c50a408380a/C of 4a01a47e5e966934386d5c50a408380a into 1185a922c7ee43b4bf5795ce00cd83c3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:25,297 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:25,297 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., storeName=4a01a47e5e966934386d5c50a408380a/C, priority=13, startTime=1732223725147; duration=0sec 2024-11-21T21:15:25,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742110_1286 (size=12151) 2024-11-21T21:15:25,297 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:25,297 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a01a47e5e966934386d5c50a408380a:C 2024-11-21T21:15:25,310 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/6c4cb720c43b424f8de2e550f9141f53 2024-11-21T21:15:25,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/c2de8df81c0344fcbe3335dc2710d193 is 50, key is test_row_0/C:col10/1732223724248/Put/seqid=0 2024-11-21T21:15:25,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742111_1287 (size=12151) 2024-11-21T21:15:25,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T21:15:25,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 4a01a47e5e966934386d5c50a408380a 2024-11-21T21:15:25,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:25,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:25,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223785417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:25,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:25,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:25,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:25,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:25,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223785419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:25,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:25,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:25,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:25,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:25,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223785424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:25,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:25,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:25,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:25,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:25,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223785426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:25,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:25,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223785426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:25,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:25,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:25,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:25,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:25,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:25,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:25,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:25,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223785532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:25,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:25,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:25,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:25,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:25,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223785532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:25,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:25,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:25,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:25,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:25,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223785540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:25,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:25,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:25,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:25,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:25,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:25,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223785541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:25,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223785541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:25,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:25,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:25,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:25,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:25,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:25,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:25,579 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/d661c5c8fb954138ba84e4f96f12a887 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/d661c5c8fb954138ba84e4f96f12a887 2024-11-21T21:15:25,583 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a01a47e5e966934386d5c50a408380a/A of 4a01a47e5e966934386d5c50a408380a into d661c5c8fb954138ba84e4f96f12a887(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:25,583 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:25,583 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., storeName=4a01a47e5e966934386d5c50a408380a/A, priority=13, startTime=1732223725146; duration=0sec 2024-11-21T21:15:25,583 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:25,583 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a01a47e5e966934386d5c50a408380a:A 2024-11-21T21:15:25,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T21:15:25,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:25,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223785740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:25,744 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:25,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:25,744 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:25,744 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:25,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223785740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:25,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:25,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:25,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:25,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:25,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223785751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:25,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:25,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:25,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:25,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223785751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:25,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:25,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:25,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:25,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:25,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:25,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223785751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:25,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:25,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:25,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:25,761 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/c2de8df81c0344fcbe3335dc2710d193 2024-11-21T21:15:25,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/b727995f8a7f46f68d7834f534373a6e as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/b727995f8a7f46f68d7834f534373a6e 2024-11-21T21:15:25,773 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/b727995f8a7f46f68d7834f534373a6e, entries=150, sequenceid=210, filesize=11.9 K 2024-11-21T21:15:25,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/6c4cb720c43b424f8de2e550f9141f53 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/6c4cb720c43b424f8de2e550f9141f53 2024-11-21T21:15:25,778 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/6c4cb720c43b424f8de2e550f9141f53, entries=150, sequenceid=210, filesize=11.9 K 2024-11-21T21:15:25,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/c2de8df81c0344fcbe3335dc2710d193 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/c2de8df81c0344fcbe3335dc2710d193 2024-11-21T21:15:25,782 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/c2de8df81c0344fcbe3335dc2710d193, entries=150, sequenceid=210, filesize=11.9 K 2024-11-21T21:15:25,783 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 4a01a47e5e966934386d5c50a408380a in 562ms, sequenceid=210, compaction requested=false 2024-11-21T21:15:25,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:25,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:25,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-21T21:15:25,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-21T21:15:25,786 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-21T21:15:25,786 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 716 msec 2024-11-21T21:15:25,788 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 726 msec 2024-11-21T21:15:26,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 4a01a47e5e966934386d5c50a408380a 2024-11-21T21:15:26,053 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a01a47e5e966934386d5c50a408380a 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-21T21:15:26,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=A 2024-11-21T21:15:26,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:26,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=B 2024-11-21T21:15:26,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:26,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=C 2024-11-21T21:15:26,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:26,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/91fd1e765e074c4eac77941394bdad3c is 50, key is test_row_0/A:col10/1732223725424/Put/seqid=0 2024-11-21T21:15:26,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742112_1288 (size=14541) 2024-11-21T21:15:26,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:26,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223786066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:26,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:26,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:26,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:26,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223786066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:26,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:26,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:26,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:26,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:26,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223786067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:26,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:26,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:26,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:26,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:26,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:26,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223786070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:26,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:26,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223786071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:26,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:26,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:26,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:26,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:26,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:26,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:26,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:26,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223786180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:26,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:26,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:26,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:26,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:26,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223786180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:26,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:26,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:26,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:26,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:26,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223786181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:26,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:26,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:26,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:26,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:26,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223786181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:26,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:26,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:26,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:26,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:26,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223786184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:26,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:26,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:26,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:26,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T21:15:26,201 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:15:26,201 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:15:26,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-21T21:15:26,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T21:15:26,204 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:15:26,207 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:15:26,207 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:15:26,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T21:15:26,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-21T21:15:26,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:26,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:26,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:26,361 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:26,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:26,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:26,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:26,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223786390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:26,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:26,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:26,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:26,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223786390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:26,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:26,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:26,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:26,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:26,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:26,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223786390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:26,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:26,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:26,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:26,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:26,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223786390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:26,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:26,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:26,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:26,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:26,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223786400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:26,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:26,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:26,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:26,463 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/91fd1e765e074c4eac77941394bdad3c 2024-11-21T21:15:26,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/7cf1924b1a8e40779c271f3e4599b2a4 is 50, key is test_row_0/B:col10/1732223725424/Put/seqid=0 2024-11-21T21:15:26,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742113_1289 (size=12151) 2024-11-21T21:15:26,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-21T21:15:26,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:26,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:26,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:26,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:26,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:26,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:26,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T21:15:26,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-21T21:15:26,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:26,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:26,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:26,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:26,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:26,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:26,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:26,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223786700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:26,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:26,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:26,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:26,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:26,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223786700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:26,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:26,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:26,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:26,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:26,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223786700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:26,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:26,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:26,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:26,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223786700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:26,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:26,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:26,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:26,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:26,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:26,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223786711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:26,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:26,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:26,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:26,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-21T21:15:26,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:26,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:26,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:26,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:26,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:26,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:26,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T21:15:26,875 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/7cf1924b1a8e40779c271f3e4599b2a4 2024-11-21T21:15:26,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/2101faf1cf31476f8202f261b5d19cdb is 50, key is test_row_0/C:col10/1732223725424/Put/seqid=0 2024-11-21T21:15:26,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742114_1290 (size=12151) 2024-11-21T21:15:26,948 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/2101faf1cf31476f8202f261b5d19cdb 2024-11-21T21:15:26,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/91fd1e765e074c4eac77941394bdad3c as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/91fd1e765e074c4eac77941394bdad3c 2024-11-21T21:15:26,963 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/91fd1e765e074c4eac77941394bdad3c, entries=200, sequenceid=231, filesize=14.2 K 2024-11-21T21:15:26,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/7cf1924b1a8e40779c271f3e4599b2a4 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/7cf1924b1a8e40779c271f3e4599b2a4 2024-11-21T21:15:26,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/7cf1924b1a8e40779c271f3e4599b2a4, entries=150, sequenceid=231, filesize=11.9 K 2024-11-21T21:15:26,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-21T21:15:26,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:26,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:26,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:26,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:26,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:26,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:26,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/2101faf1cf31476f8202f261b5d19cdb as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/2101faf1cf31476f8202f261b5d19cdb 2024-11-21T21:15:26,994 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/2101faf1cf31476f8202f261b5d19cdb, entries=150, sequenceid=231, filesize=11.9 K 2024-11-21T21:15:26,995 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 4a01a47e5e966934386d5c50a408380a in 942ms, sequenceid=231, compaction requested=true 2024-11-21T21:15:26,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:26,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a01a47e5e966934386d5c50a408380a:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:15:26,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:26,995 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:15:26,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a01a47e5e966934386d5c50a408380a:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:15:26,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:15:26,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a01a47e5e966934386d5c50a408380a:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:15:26,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T21:15:26,995 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:15:26,996 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39355 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:15:26,997 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): 4a01a47e5e966934386d5c50a408380a/A is initiating minor compaction (all files) 2024-11-21T21:15:26,997 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a01a47e5e966934386d5c50a408380a/A in TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:26,997 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/d661c5c8fb954138ba84e4f96f12a887, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/b727995f8a7f46f68d7834f534373a6e, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/91fd1e765e074c4eac77941394bdad3c] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp, totalSize=38.4 K 2024-11-21T21:15:26,998 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:15:26,998 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): 4a01a47e5e966934386d5c50a408380a/B is initiating minor compaction (all files) 2024-11-21T21:15:26,998 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a01a47e5e966934386d5c50a408380a/B in TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:26,999 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/6b28d618aa2b4c349314ab6a45b913fd, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/6c4cb720c43b424f8de2e550f9141f53, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/7cf1924b1a8e40779c271f3e4599b2a4] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp, totalSize=36.1 K 2024-11-21T21:15:26,999 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting d661c5c8fb954138ba84e4f96f12a887, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732223723591 2024-11-21T21:15:26,999 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b28d618aa2b4c349314ab6a45b913fd, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732223723591 2024-11-21T21:15:26,999 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting b727995f8a7f46f68d7834f534373a6e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732223724248 2024-11-21T21:15:27,000 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c4cb720c43b424f8de2e550f9141f53, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732223724248 2024-11-21T21:15:27,000 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 7cf1924b1a8e40779c271f3e4599b2a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732223725424 2024-11-21T21:15:27,000 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91fd1e765e074c4eac77941394bdad3c, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732223725411 2024-11-21T21:15:27,033 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a01a47e5e966934386d5c50a408380a#B#compaction#252 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:27,033 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/b7bacde0e1fd4fbf8743fbc8c7d2f476 is 50, key is test_row_0/B:col10/1732223725424/Put/seqid=0 2024-11-21T21:15:27,054 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a01a47e5e966934386d5c50a408380a#A#compaction#253 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:27,054 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/788f15e5ecbf4ad2980a5f0595957f26 is 50, key is test_row_0/A:col10/1732223725424/Put/seqid=0 2024-11-21T21:15:27,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742115_1291 (size=12765) 2024-11-21T21:15:27,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-21T21:15:27,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:27,128 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 4a01a47e5e966934386d5c50a408380a 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-21T21:15:27,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=A 2024-11-21T21:15:27,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:27,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=B 2024-11-21T21:15:27,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:27,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=C 2024-11-21T21:15:27,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:27,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742116_1292 (size=12765) 2024-11-21T21:15:27,142 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/788f15e5ecbf4ad2980a5f0595957f26 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/788f15e5ecbf4ad2980a5f0595957f26 2024-11-21T21:15:27,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/f9437f9f78f64e139a2139e452548dd2 is 50, key is test_row_0/A:col10/1732223726066/Put/seqid=0 2024-11-21T21:15:27,155 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a01a47e5e966934386d5c50a408380a/A of 4a01a47e5e966934386d5c50a408380a into 788f15e5ecbf4ad2980a5f0595957f26(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:27,155 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:27,155 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., storeName=4a01a47e5e966934386d5c50a408380a/A, priority=13, startTime=1732223726995; duration=0sec 2024-11-21T21:15:27,155 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:15:27,155 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a01a47e5e966934386d5c50a408380a:A 2024-11-21T21:15:27,155 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:15:27,157 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:15:27,157 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): 4a01a47e5e966934386d5c50a408380a/C is initiating minor compaction (all files) 2024-11-21T21:15:27,157 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a01a47e5e966934386d5c50a408380a/C in TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:27,157 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/1185a922c7ee43b4bf5795ce00cd83c3, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/c2de8df81c0344fcbe3335dc2710d193, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/2101faf1cf31476f8202f261b5d19cdb] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp, totalSize=36.1 K 2024-11-21T21:15:27,158 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1185a922c7ee43b4bf5795ce00cd83c3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732223723591 2024-11-21T21:15:27,158 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2de8df81c0344fcbe3335dc2710d193, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732223724248 2024-11-21T21:15:27,158 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2101faf1cf31476f8202f261b5d19cdb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732223725424 2024-11-21T21:15:27,225 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a01a47e5e966934386d5c50a408380a#C#compaction#255 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:27,225 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/63b0514f2a084dfbaad1801e1c735726 is 50, key is test_row_0/C:col10/1732223725424/Put/seqid=0 2024-11-21T21:15:27,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:27,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742117_1293 (size=12151) 2024-11-21T21:15:27,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 4a01a47e5e966934386d5c50a408380a 2024-11-21T21:15:27,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742118_1294 (size=12765) 2024-11-21T21:15:27,252 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/63b0514f2a084dfbaad1801e1c735726 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/63b0514f2a084dfbaad1801e1c735726 2024-11-21T21:15:27,258 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a01a47e5e966934386d5c50a408380a/C of 4a01a47e5e966934386d5c50a408380a into 63b0514f2a084dfbaad1801e1c735726(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:27,258 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:27,258 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., storeName=4a01a47e5e966934386d5c50a408380a/C, priority=13, startTime=1732223726995; duration=0sec 2024-11-21T21:15:27,258 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:27,258 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a01a47e5e966934386d5c50a408380a:C 2024-11-21T21:15:27,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:27,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223787291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:27,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:27,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:27,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223787291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:27,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:27,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:27,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:27,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:27,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:27,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223787291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:27,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:27,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:27,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:27,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:27,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:27,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223787292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:27,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:27,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:27,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:27,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:27,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223787297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:27,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:27,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:27,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:27,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T21:15:27,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:27,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223787400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:27,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:27,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223787400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:27,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:27,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:27,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223787400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:27,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:27,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:27,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:27,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:27,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:27,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:27,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:27,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:27,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:27,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223787400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:27,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:27,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:27,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:27,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:27,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223787410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:27,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:27,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:27,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:27,519 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/b7bacde0e1fd4fbf8743fbc8c7d2f476 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/b7bacde0e1fd4fbf8743fbc8c7d2f476 2024-11-21T21:15:27,524 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a01a47e5e966934386d5c50a408380a/B of 4a01a47e5e966934386d5c50a408380a into b7bacde0e1fd4fbf8743fbc8c7d2f476(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:27,524 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:27,524 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., storeName=4a01a47e5e966934386d5c50a408380a/B, priority=13, startTime=1732223726995; duration=0sec 2024-11-21T21:15:27,524 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:27,524 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a01a47e5e966934386d5c50a408380a:B 2024-11-21T21:15:27,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:27,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223787610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:27,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:27,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:27,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223787610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:27,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:27,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:27,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:27,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:27,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:27,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:27,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223787610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:27,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:27,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:27,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:27,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:27,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223787610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:27,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:27,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:27,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:27,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:27,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223787620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:27,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:27,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:27,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:27,739 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/f9437f9f78f64e139a2139e452548dd2 2024-11-21T21:15:27,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/3dc203bbdf984efaac203f7205869a6f is 50, key is test_row_0/B:col10/1732223726066/Put/seqid=0 2024-11-21T21:15:27,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742119_1295 (size=12151) 2024-11-21T21:15:27,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:27,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223787921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:27,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:27,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:27,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:27,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:27,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223787921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:27,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:27,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:27,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:27,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:27,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223787921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:27,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:27,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223787921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:27,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:27,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:27,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:27,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:27,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:27,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:27,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:27,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223787930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:27,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:27,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:27,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:28,154 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/3dc203bbdf984efaac203f7205869a6f 2024-11-21T21:15:28,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/66552d099e3c4f44aeb832395a13f4c8 is 50, key is test_row_0/C:col10/1732223726066/Put/seqid=0 2024-11-21T21:15:28,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742120_1296 (size=12151) 2024-11-21T21:15:28,166 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/66552d099e3c4f44aeb832395a13f4c8 2024-11-21T21:15:28,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/f9437f9f78f64e139a2139e452548dd2 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/f9437f9f78f64e139a2139e452548dd2 2024-11-21T21:15:28,188 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/f9437f9f78f64e139a2139e452548dd2, entries=150, sequenceid=244, filesize=11.9 K 2024-11-21T21:15:28,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/3dc203bbdf984efaac203f7205869a6f as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/3dc203bbdf984efaac203f7205869a6f 2024-11-21T21:15:28,196 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/3dc203bbdf984efaac203f7205869a6f, entries=150, sequenceid=244, filesize=11.9 K 2024-11-21T21:15:28,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/66552d099e3c4f44aeb832395a13f4c8 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/66552d099e3c4f44aeb832395a13f4c8 2024-11-21T21:15:28,203 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/66552d099e3c4f44aeb832395a13f4c8, entries=150, sequenceid=244, filesize=11.9 K 2024-11-21T21:15:28,204 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 4a01a47e5e966934386d5c50a408380a in 1076ms, sequenceid=244, compaction requested=false 2024-11-21T21:15:28,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:28,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:28,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-21T21:15:28,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-21T21:15:28,207 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-21T21:15:28,207 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9980 sec 2024-11-21T21:15:28,209 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 2.0070 sec 2024-11-21T21:15:28,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T21:15:28,351 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:15:28,351 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:15:28,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-21T21:15:28,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-21T21:15:28,353 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:15:28,353 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:15:28,353 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:15:28,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 4a01a47e5e966934386d5c50a408380a 2024-11-21T21:15:28,433 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a01a47e5e966934386d5c50a408380a 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-21T21:15:28,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=A 2024-11-21T21:15:28,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:28,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=B 2024-11-21T21:15:28,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:28,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=C 2024-11-21T21:15:28,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:28,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/84e10c37a1c345b68bfb079dee8b309f is 50, key is test_row_0/A:col10/1732223728432/Put/seqid=0 2024-11-21T21:15:28,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742121_1297 (size=12301) 2024-11-21T21:15:28,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:28,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223788442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:28,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:28,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:28,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223788443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:28,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:28,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:28,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:28,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:28,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:28,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:28,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223788443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:28,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:28,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:28,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:28,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223788446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:28,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:28,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:28,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:28,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:28,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:28,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223788448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:28,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:28,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:28,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:28,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-21T21:15:28,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-21T21:15:28,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:28,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:28,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:28,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:28,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:28,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:28,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:28,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223788550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:28,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:28,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:28,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:28,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:28,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223788560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:28,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:28,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:28,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223788560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:28,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:28,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:28,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:28,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:28,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:28,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:28,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223788560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:28,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:28,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:28,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:28,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:28,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223788570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:28,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:28,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:28,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:28,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-21T21:15:28,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:28,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:28,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:28,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:28,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:28,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:28,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-21T21:15:28,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:28,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223788760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:28,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:28,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:28,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:28,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:28,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223788770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:28,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:28,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:28,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:28,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:28,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223788771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:28,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:28,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:28,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:28,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:28,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223788771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:28,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:28,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:28,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:28,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:28,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223788780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:28,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:28,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:28,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:28,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-21T21:15:28,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:28,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:28,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:28,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:28,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:28,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:28,845 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/84e10c37a1c345b68bfb079dee8b309f 2024-11-21T21:15:28,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/1fbc739d6f0f4128bb2eac5db6933e4f is 50, key is test_row_0/B:col10/1732223728432/Put/seqid=0 2024-11-21T21:15:28,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742122_1298 (size=12301) 2024-11-21T21:15:28,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-21T21:15:28,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:28,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:28,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:28,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:28,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:28,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:28,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-21T21:15:29,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:29,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223789070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:29,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:29,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:29,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:29,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:29,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223789080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:29,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:29,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:29,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:29,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:29,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223789081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:29,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:29,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:29,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223789081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:29,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:29,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:29,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:29,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:29,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:29,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:29,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223789090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:29,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:29,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:29,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:29,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-21T21:15:29,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:29,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:29,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:29,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:29,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:29,258 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/1fbc739d6f0f4128bb2eac5db6933e4f 2024-11-21T21:15:29,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/f044800eb8534605a24ea606968811be is 50, key is test_row_0/C:col10/1732223728432/Put/seqid=0 2024-11-21T21:15:29,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-21T21:15:29,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:29,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:29,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:29,269 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:29,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:29,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742123_1299 (size=12301) 2024-11-21T21:15:29,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:29,271 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/f044800eb8534605a24ea606968811be 2024-11-21T21:15:29,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/84e10c37a1c345b68bfb079dee8b309f as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/84e10c37a1c345b68bfb079dee8b309f 2024-11-21T21:15:29,283 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/84e10c37a1c345b68bfb079dee8b309f, entries=150, sequenceid=267, filesize=12.0 K 2024-11-21T21:15:29,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/1fbc739d6f0f4128bb2eac5db6933e4f as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/1fbc739d6f0f4128bb2eac5db6933e4f 2024-11-21T21:15:29,288 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/1fbc739d6f0f4128bb2eac5db6933e4f, entries=150, sequenceid=267, filesize=12.0 K 2024-11-21T21:15:29,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/f044800eb8534605a24ea606968811be as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/f044800eb8534605a24ea606968811be 2024-11-21T21:15:29,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/f044800eb8534605a24ea606968811be, entries=150, sequenceid=267, filesize=12.0 K 2024-11-21T21:15:29,294 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 4a01a47e5e966934386d5c50a408380a in 861ms, sequenceid=267, compaction requested=true 2024-11-21T21:15:29,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:29,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a01a47e5e966934386d5c50a408380a:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:15:29,294 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:15:29,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:29,294 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:15:29,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a01a47e5e966934386d5c50a408380a:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:15:29,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:29,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a01a47e5e966934386d5c50a408380a:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:15:29,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:15:29,295 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37217 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:15:29,295 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37217 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:15:29,295 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): 4a01a47e5e966934386d5c50a408380a/A is initiating minor compaction (all files) 2024-11-21T21:15:29,295 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): 4a01a47e5e966934386d5c50a408380a/B is initiating minor compaction (all files) 2024-11-21T21:15:29,295 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a01a47e5e966934386d5c50a408380a/A in TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:29,295 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a01a47e5e966934386d5c50a408380a/B in TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:29,295 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/b7bacde0e1fd4fbf8743fbc8c7d2f476, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/3dc203bbdf984efaac203f7205869a6f, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/1fbc739d6f0f4128bb2eac5db6933e4f] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp, totalSize=36.3 K 2024-11-21T21:15:29,295 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/788f15e5ecbf4ad2980a5f0595957f26, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/f9437f9f78f64e139a2139e452548dd2, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/84e10c37a1c345b68bfb079dee8b309f] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp, totalSize=36.3 K 2024-11-21T21:15:29,296 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting b7bacde0e1fd4fbf8743fbc8c7d2f476, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732223725424 2024-11-21T21:15:29,296 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 788f15e5ecbf4ad2980a5f0595957f26, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732223725424 2024-11-21T21:15:29,296 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 3dc203bbdf984efaac203f7205869a6f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1732223726065 2024-11-21T21:15:29,296 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9437f9f78f64e139a2139e452548dd2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1732223726065 2024-11-21T21:15:29,296 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fbc739d6f0f4128bb2eac5db6933e4f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732223727266 2024-11-21T21:15:29,296 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84e10c37a1c345b68bfb079dee8b309f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732223727266 2024-11-21T21:15:29,306 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a01a47e5e966934386d5c50a408380a#B#compaction#261 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:29,306 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/bc58f85f41334cbfb7a845ab8966cd82 is 50, key is test_row_0/B:col10/1732223728432/Put/seqid=0 2024-11-21T21:15:29,311 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a01a47e5e966934386d5c50a408380a#A#compaction#262 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:29,311 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/6adc7ef1edb44c26b93503a4b6db3717 is 50, key is test_row_0/A:col10/1732223728432/Put/seqid=0 2024-11-21T21:15:29,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742124_1300 (size=13017) 2024-11-21T21:15:29,323 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/bc58f85f41334cbfb7a845ab8966cd82 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/bc58f85f41334cbfb7a845ab8966cd82 2024-11-21T21:15:29,328 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a01a47e5e966934386d5c50a408380a/B of 4a01a47e5e966934386d5c50a408380a into bc58f85f41334cbfb7a845ab8966cd82(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:29,329 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:29,329 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., storeName=4a01a47e5e966934386d5c50a408380a/B, priority=13, startTime=1732223729294; duration=0sec 2024-11-21T21:15:29,329 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:15:29,329 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a01a47e5e966934386d5c50a408380a:B 2024-11-21T21:15:29,329 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:15:29,330 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37217 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:15:29,330 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): 4a01a47e5e966934386d5c50a408380a/C is initiating minor compaction (all files) 2024-11-21T21:15:29,330 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a01a47e5e966934386d5c50a408380a/C in TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:29,330 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/63b0514f2a084dfbaad1801e1c735726, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/66552d099e3c4f44aeb832395a13f4c8, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/f044800eb8534605a24ea606968811be] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp, totalSize=36.3 K 2024-11-21T21:15:29,330 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 63b0514f2a084dfbaad1801e1c735726, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732223725424 2024-11-21T21:15:29,331 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 66552d099e3c4f44aeb832395a13f4c8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1732223726065 2024-11-21T21:15:29,331 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting f044800eb8534605a24ea606968811be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732223727266 2024-11-21T21:15:29,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742125_1301 (size=13017) 2024-11-21T21:15:29,346 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/6adc7ef1edb44c26b93503a4b6db3717 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/6adc7ef1edb44c26b93503a4b6db3717 2024-11-21T21:15:29,354 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a01a47e5e966934386d5c50a408380a#C#compaction#263 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:29,354 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/d02926a172834ebaa159bd9aa5ad27af is 50, key is test_row_0/C:col10/1732223728432/Put/seqid=0 2024-11-21T21:15:29,359 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a01a47e5e966934386d5c50a408380a/A of 4a01a47e5e966934386d5c50a408380a into 6adc7ef1edb44c26b93503a4b6db3717(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:29,359 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:29,359 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., storeName=4a01a47e5e966934386d5c50a408380a/A, priority=13, startTime=1732223729294; duration=0sec 2024-11-21T21:15:29,359 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:29,359 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a01a47e5e966934386d5c50a408380a:A 2024-11-21T21:15:29,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742126_1302 (size=13017) 2024-11-21T21:15:29,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-21T21:15:29,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:29,423 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 4a01a47e5e966934386d5c50a408380a 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-21T21:15:29,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=A 2024-11-21T21:15:29,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:29,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=B 2024-11-21T21:15:29,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:29,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=C 2024-11-21T21:15:29,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:29,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/66b42356cf3044d8bc8b0d46953480c4 is 50, key is test_row_0/A:col10/1732223728440/Put/seqid=0 2024-11-21T21:15:29,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742127_1303 (size=12301) 2024-11-21T21:15:29,436 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/66b42356cf3044d8bc8b0d46953480c4 2024-11-21T21:15:29,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/8257097dd9044a65ae21265da2228663 is 50, key is test_row_0/B:col10/1732223728440/Put/seqid=0 2024-11-21T21:15:29,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742128_1304 (size=12301) 2024-11-21T21:15:29,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-21T21:15:29,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 4a01a47e5e966934386d5c50a408380a 2024-11-21T21:15:29,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:29,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:29,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223789614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:29,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732223729614, max timestamp=1732223729619Num uniques 859; 2024-11-21T21:15:29,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732223729614, max timestamp=1732223729619Num uniques 859; 2024-11-21T21:15:29,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:29,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T21:15:29,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223789615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:29,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T21:15:29,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:29,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:29,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:29,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:29,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:29,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:29,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732223729614, max timestamp=1732223729619Num uniques 859; 2024-11-21T21:15:29,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T21:15:29,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:29,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223789649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:29,657 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:29,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:29,657 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:29,657 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:29,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223789653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:29,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:29,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:29,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:29,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:29,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223789657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:29,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:29,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:29,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:29,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:29,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223789730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:29,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:29,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223789731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:29,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:29,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:29,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:29,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:29,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:29,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:29,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:29,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223789760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:29,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:29,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:29,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:29,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:29,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223789760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:29,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:29,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:29,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:29,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:29,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223789770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:29,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:29,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:29,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:29,785 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/d02926a172834ebaa159bd9aa5ad27af as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/d02926a172834ebaa159bd9aa5ad27af 2024-11-21T21:15:29,791 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a01a47e5e966934386d5c50a408380a/C of 4a01a47e5e966934386d5c50a408380a into d02926a172834ebaa159bd9aa5ad27af(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:29,792 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:29,792 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., storeName=4a01a47e5e966934386d5c50a408380a/C, priority=13, startTime=1732223729294; duration=0sec 2024-11-21T21:15:29,792 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:29,792 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a01a47e5e966934386d5c50a408380a:C 2024-11-21T21:15:29,848 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/8257097dd9044a65ae21265da2228663 2024-11-21T21:15:29,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/7170f3aef45444819eff3a344734acc9 is 50, key is test_row_0/C:col10/1732223728440/Put/seqid=0 2024-11-21T21:15:29,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742129_1305 (size=12301) 2024-11-21T21:15:29,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:29,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223789940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:29,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:29,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:29,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223789940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:29,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:29,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:29,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:29,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:29,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:29,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:29,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223789970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:29,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:29,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223789970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:29,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:29,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:29,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:29,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:29,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:29,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:29,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:29,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223789980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:29,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:29,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:29,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:30,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:30,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223790254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:30,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:30,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223790254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:30,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:30,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:30,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:30,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:30,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:30,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:30,260 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/7170f3aef45444819eff3a344734acc9 2024-11-21T21:15:30,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/66b42356cf3044d8bc8b0d46953480c4 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/66b42356cf3044d8bc8b0d46953480c4 2024-11-21T21:15:30,271 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/66b42356cf3044d8bc8b0d46953480c4, entries=150, sequenceid=281, filesize=12.0 K 2024-11-21T21:15:30,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/8257097dd9044a65ae21265da2228663 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/8257097dd9044a65ae21265da2228663 2024-11-21T21:15:30,276 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/8257097dd9044a65ae21265da2228663, entries=150, sequenceid=281, filesize=12.0 K 2024-11-21T21:15:30,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/7170f3aef45444819eff3a344734acc9 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/7170f3aef45444819eff3a344734acc9 2024-11-21T21:15:30,281 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/7170f3aef45444819eff3a344734acc9, entries=150, sequenceid=281, filesize=12.0 K 2024-11-21T21:15:30,283 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 4a01a47e5e966934386d5c50a408380a in 860ms, sequenceid=281, compaction requested=false 2024-11-21T21:15:30,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:30,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:30,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-21T21:15:30,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-21T21:15:30,287 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-21T21:15:30,287 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9320 sec 2024-11-21T21:15:30,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 4a01a47e5e966934386d5c50a408380a 2024-11-21T21:15:30,288 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a01a47e5e966934386d5c50a408380a 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-21T21:15:30,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=A 2024-11-21T21:15:30,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:15:30,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=B 2024-11-21T21:15:30,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:15:30,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=C 2024-11-21T21:15:30,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:15:30,290 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.9380 sec 2024-11-21T21:15:30,293 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/f7fa999fbdd94c61a31b30c6fc40b17e is 50, key is test_row_0/A:col10/1732223729632/Put/seqid=0 2024-11-21T21:15:30,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:30,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223790309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:30,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:30,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:30,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:30,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742130_1306 (size=14741) 2024-11-21T21:15:30,317 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/f7fa999fbdd94c61a31b30c6fc40b17e 2024-11-21T21:15:30,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:30,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223790316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:30,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:30,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223790316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:30,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:30,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:30,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:30,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/40e9ec4e721c4710b6a6d8b1853bfdb6 is 50, key is test_row_0/B:col10/1732223729632/Put/seqid=0 2024-11-21T21:15:30,326 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:30,326 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:30,326 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:30,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742131_1307 (size=12301) 2024-11-21T21:15:30,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:30,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223790420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:30,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:30,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:30,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:30,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:30,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223790430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:30,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:30,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:30,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:30,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:30,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223790430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:30,435 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:30,435 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:30,435 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:30,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-21T21:15:30,500 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:15:30,501 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:15:30,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-21T21:15:30,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-21T21:15:30,502 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:15:30,503 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:15:30,503 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:15:30,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-21T21:15:30,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:30,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223790630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:30,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:30,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:30,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:30,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:30,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223790640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:30,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:30,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:30,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:30,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:30,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223790640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:30,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:30,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:30,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:30,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-21T21:15:30,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:30,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:30,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:30,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:30,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:30,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:30,742 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/40e9ec4e721c4710b6a6d8b1853bfdb6 2024-11-21T21:15:30,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/6d18fcb42076447085636c979ac93a9a is 50, key is test_row_0/C:col10/1732223729632/Put/seqid=0 2024-11-21T21:15:30,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742132_1308 (size=12301) 2024-11-21T21:15:30,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:30,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223790770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:30,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:30,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:30,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:30,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:30,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223790770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:30,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:30,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:30,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:30,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-21T21:15:30,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:30,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:30,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:30,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:30,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:30,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:30,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-21T21:15:30,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:30,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223790940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:30,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:30,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:30,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:30,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:30,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223790950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:30,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:30,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:30,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:30,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:30,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223790951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:30,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:30,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:30,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:30,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-21T21:15:30,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:30,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:30,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:30,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:30,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:30,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:31,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-21T21:15:31,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:31,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:31,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:31,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:31,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:31,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:31,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-21T21:15:31,159 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/6d18fcb42076447085636c979ac93a9a 2024-11-21T21:15:31,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/f7fa999fbdd94c61a31b30c6fc40b17e as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/f7fa999fbdd94c61a31b30c6fc40b17e 2024-11-21T21:15:31,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/f7fa999fbdd94c61a31b30c6fc40b17e, entries=200, sequenceid=318, filesize=14.4 K 2024-11-21T21:15:31,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/40e9ec4e721c4710b6a6d8b1853bfdb6 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/40e9ec4e721c4710b6a6d8b1853bfdb6 2024-11-21T21:15:31,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/40e9ec4e721c4710b6a6d8b1853bfdb6, entries=150, sequenceid=318, filesize=12.0 K 2024-11-21T21:15:31,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/6d18fcb42076447085636c979ac93a9a as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/6d18fcb42076447085636c979ac93a9a 2024-11-21T21:15:31,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/6d18fcb42076447085636c979ac93a9a, entries=150, sequenceid=318, filesize=12.0 K 2024-11-21T21:15:31,173 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=60.38 KB/61830 for 4a01a47e5e966934386d5c50a408380a in 885ms, sequenceid=318, compaction requested=true 2024-11-21T21:15:31,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:31,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a01a47e5e966934386d5c50a408380a:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:15:31,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:31,173 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:15:31,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a01a47e5e966934386d5c50a408380a:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:15:31,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:31,173 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:15:31,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a01a47e5e966934386d5c50a408380a:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:15:31,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:15:31,174 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:15:31,174 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:15:31,174 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): 4a01a47e5e966934386d5c50a408380a/A is initiating minor compaction (all files) 2024-11-21T21:15:31,174 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): 4a01a47e5e966934386d5c50a408380a/B is initiating minor compaction (all files) 2024-11-21T21:15:31,174 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a01a47e5e966934386d5c50a408380a/B in TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:31,174 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a01a47e5e966934386d5c50a408380a/A in TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:31,174 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/bc58f85f41334cbfb7a845ab8966cd82, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/8257097dd9044a65ae21265da2228663, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/40e9ec4e721c4710b6a6d8b1853bfdb6] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp, totalSize=36.7 K 2024-11-21T21:15:31,174 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/6adc7ef1edb44c26b93503a4b6db3717, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/66b42356cf3044d8bc8b0d46953480c4, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/f7fa999fbdd94c61a31b30c6fc40b17e] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp, totalSize=39.1 K 2024-11-21T21:15:31,175 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting bc58f85f41334cbfb7a845ab8966cd82, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732223727266 2024-11-21T21:15:31,175 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 8257097dd9044a65ae21265da2228663, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732223728440 2024-11-21T21:15:31,175 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6adc7ef1edb44c26b93503a4b6db3717, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732223727266 2024-11-21T21:15:31,176 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66b42356cf3044d8bc8b0d46953480c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732223728440 2024-11-21T21:15:31,176 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 40e9ec4e721c4710b6a6d8b1853bfdb6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1732223729632 2024-11-21T21:15:31,176 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7fa999fbdd94c61a31b30c6fc40b17e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1732223729632 2024-11-21T21:15:31,183 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a01a47e5e966934386d5c50a408380a#A#compaction#271 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:31,183 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a01a47e5e966934386d5c50a408380a#B#compaction#270 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:31,183 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/774676ec24d04102b570ae24617068d6 is 50, key is test_row_0/A:col10/1732223729632/Put/seqid=0 2024-11-21T21:15:31,183 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/7281d436b7b04f6993639981e0eb88b3 is 50, key is test_row_0/B:col10/1732223729632/Put/seqid=0 2024-11-21T21:15:31,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742133_1309 (size=13119) 2024-11-21T21:15:31,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742134_1310 (size=13119) 2024-11-21T21:15:31,196 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/7281d436b7b04f6993639981e0eb88b3 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/7281d436b7b04f6993639981e0eb88b3 2024-11-21T21:15:31,201 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a01a47e5e966934386d5c50a408380a/B of 4a01a47e5e966934386d5c50a408380a into 7281d436b7b04f6993639981e0eb88b3(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:31,201 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:31,201 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., storeName=4a01a47e5e966934386d5c50a408380a/B, priority=13, startTime=1732223731173; duration=0sec 2024-11-21T21:15:31,201 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:15:31,201 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a01a47e5e966934386d5c50a408380a:B 2024-11-21T21:15:31,201 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:15:31,202 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:15:31,202 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): 4a01a47e5e966934386d5c50a408380a/C is initiating minor compaction (all files) 2024-11-21T21:15:31,202 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a01a47e5e966934386d5c50a408380a/C in TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:31,202 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/d02926a172834ebaa159bd9aa5ad27af, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/7170f3aef45444819eff3a344734acc9, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/6d18fcb42076447085636c979ac93a9a] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp, totalSize=36.7 K 2024-11-21T21:15:31,202 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting d02926a172834ebaa159bd9aa5ad27af, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732223727266 2024-11-21T21:15:31,204 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 7170f3aef45444819eff3a344734acc9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732223728440 2024-11-21T21:15:31,204 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d18fcb42076447085636c979ac93a9a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1732223729632 2024-11-21T21:15:31,209 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a01a47e5e966934386d5c50a408380a#C#compaction#272 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:31,210 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/89c3dba9870f4b7689e35474afa5e79b is 50, key is test_row_0/C:col10/1732223729632/Put/seqid=0 2024-11-21T21:15:31,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742135_1311 (size=13119) 2024-11-21T21:15:31,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-21T21:15:31,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:31,270 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 4a01a47e5e966934386d5c50a408380a 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-21T21:15:31,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=A 2024-11-21T21:15:31,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:31,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=B 2024-11-21T21:15:31,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:31,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=C 2024-11-21T21:15:31,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:31,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/64f2c7edaef0402ea70f77c6fcad0736 is 50, key is test_row_0/A:col10/1732223730305/Put/seqid=0 2024-11-21T21:15:31,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742136_1312 (size=12301) 2024-11-21T21:15:31,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 4a01a47e5e966934386d5c50a408380a 2024-11-21T21:15:31,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:31,487 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732223731477, max timestamp=1732223731484Num uniques 859; 2024-11-21T21:15:31,487 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T21:15:31,487 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732223731477, max timestamp=1732223731484Num uniques 859; 2024-11-21T21:15:31,487 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732223731477, max timestamp=1732223731484Num uniques 859; 2024-11-21T21:15:31,487 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T21:15:31,488 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T21:15:31,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:31,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223791510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:31,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:31,516 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:31,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223791511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:31,516 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:31,516 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:31,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:31,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:31,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:31,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:31,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223791511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:31,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:31,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:31,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:31,595 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/774676ec24d04102b570ae24617068d6 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/774676ec24d04102b570ae24617068d6 2024-11-21T21:15:31,600 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a01a47e5e966934386d5c50a408380a/A of 4a01a47e5e966934386d5c50a408380a into 774676ec24d04102b570ae24617068d6(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:31,600 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:31,600 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., storeName=4a01a47e5e966934386d5c50a408380a/A, priority=13, startTime=1732223731173; duration=0sec 2024-11-21T21:15:31,600 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:31,601 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a01a47e5e966934386d5c50a408380a:A 2024-11-21T21:15:31,619 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/89c3dba9870f4b7689e35474afa5e79b as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/89c3dba9870f4b7689e35474afa5e79b 2024-11-21T21:15:31,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:31,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:31,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223791621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:31,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223791622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:31,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:31,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:31,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223791622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:31,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:31,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:31,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:31,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:31,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:31,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:31,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:31,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:31,631 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a01a47e5e966934386d5c50a408380a/C of 4a01a47e5e966934386d5c50a408380a into 89c3dba9870f4b7689e35474afa5e79b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:31,631 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:31,631 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., storeName=4a01a47e5e966934386d5c50a408380a/C, priority=13, startTime=1732223731173; duration=0sec 2024-11-21T21:15:31,631 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:31,631 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a01a47e5e966934386d5c50a408380a:C 2024-11-21T21:15:31,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-21T21:15:31,680 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/64f2c7edaef0402ea70f77c6fcad0736 2024-11-21T21:15:31,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/ff1e6dda46f84b15837f8f46c634acd9 is 50, key is test_row_0/B:col10/1732223730305/Put/seqid=0 2024-11-21T21:15:31,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742137_1313 (size=12301) 2024-11-21T21:15:31,699 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/ff1e6dda46f84b15837f8f46c634acd9 2024-11-21T21:15:31,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/f95ffed44a6047919d1fd1cda00aabc9 is 50, key is test_row_0/C:col10/1732223730305/Put/seqid=0 2024-11-21T21:15:31,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742138_1314 (size=12301) 2024-11-21T21:15:31,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:31,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223791790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:31,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:31,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223791790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:31,794 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,41213,1732223653469 for 'test_row_0' in 4a01a47e5e966934386d5c50a408380a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:15:31,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:31,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:31,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:31,794 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,41213,1732223653469 for 'test_row_0' in 4a01a47e5e966934386d5c50a408380a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:15:31,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:31,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:31,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:31,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:31,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223791830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:31,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:31,835 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:31,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223791830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:31,835 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:31,835 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:31,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:31,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223791831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:31,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:31,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:31,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:31,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:31,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:31,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:32,117 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/f95ffed44a6047919d1fd1cda00aabc9 2024-11-21T21:15:32,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/64f2c7edaef0402ea70f77c6fcad0736 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/64f2c7edaef0402ea70f77c6fcad0736 2024-11-21T21:15:32,130 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/64f2c7edaef0402ea70f77c6fcad0736, entries=150, sequenceid=331, filesize=12.0 K 2024-11-21T21:15:32,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/ff1e6dda46f84b15837f8f46c634acd9 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/ff1e6dda46f84b15837f8f46c634acd9 2024-11-21T21:15:32,136 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/ff1e6dda46f84b15837f8f46c634acd9, entries=150, sequenceid=331, filesize=12.0 K 2024-11-21T21:15:32,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/f95ffed44a6047919d1fd1cda00aabc9 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/f95ffed44a6047919d1fd1cda00aabc9 2024-11-21T21:15:32,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:32,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223792140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:32,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:32,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:32,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:32,143 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/f95ffed44a6047919d1fd1cda00aabc9, entries=150, sequenceid=331, filesize=12.0 K 2024-11-21T21:15:32,144 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for 4a01a47e5e966934386d5c50a408380a in 873ms, sequenceid=331, compaction requested=false 2024-11-21T21:15:32,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:32,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:32,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-21T21:15:32,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-21T21:15:32,147 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-21T21:15:32,147 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6420 sec 2024-11-21T21:15:32,148 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.6460 sec 2024-11-21T21:15:32,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 4a01a47e5e966934386d5c50a408380a 2024-11-21T21:15:32,152 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a01a47e5e966934386d5c50a408380a 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-21T21:15:32,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=A 2024-11-21T21:15:32,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:15:32,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=B 2024-11-21T21:15:32,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:15:32,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=C 2024-11-21T21:15:32,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:15:32,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/717540fd17ef4d9fb820b2516c04e2b4 is 50, key is test_row_0/A:col10/1732223731504/Put/seqid=0 2024-11-21T21:15:32,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742139_1315 (size=14741) 2024-11-21T21:15:32,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:32,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223792173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:32,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:32,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:32,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:32,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223792173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:32,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:32,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:32,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:32,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:32,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:32,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223792290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:32,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:32,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:32,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223792290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:32,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:32,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:32,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:32,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:32,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:32,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:32,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223792500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:32,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:32,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223792500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:32,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:32,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:32,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:32,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:32,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:32,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:32,567 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/717540fd17ef4d9fb820b2516c04e2b4 2024-11-21T21:15:32,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/1ac37caa6e9445fc95f3040dec2b9cd0 is 50, key is test_row_0/B:col10/1732223731504/Put/seqid=0 2024-11-21T21:15:32,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742140_1316 (size=12301) 2024-11-21T21:15:32,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-21T21:15:32,651 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:15:32,651 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:15:32,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-21T21:15:32,653 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:15:32,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-21T21:15:32,653 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:15:32,653 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:15:32,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:32,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223792651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:32,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:32,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:32,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:32,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-21T21:15:32,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-21T21:15:32,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:32,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:32,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:32,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:32,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:32,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:32,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:32,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223792810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:32,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:32,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:32,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:32,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:32,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223792811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:32,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:32,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:32,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:32,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-21T21:15:32,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:32,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:32,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:32,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:32,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:32,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:32,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-21T21:15:32,983 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/1ac37caa6e9445fc95f3040dec2b9cd0 2024-11-21T21:15:32,989 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/89eb9f60e9e2423da1cfc49c174f43b6 is 50, key is test_row_0/C:col10/1732223731504/Put/seqid=0 2024-11-21T21:15:32,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742141_1317 (size=12301) 2024-11-21T21:15:33,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-21T21:15:33,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:33,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:33,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:33,114 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:33,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:33,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:33,266 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-21T21:15:33,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:33,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:33,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:33,266 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:33,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:33,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:33,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-21T21:15:33,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:33,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223793320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:33,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:33,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:33,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223793320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:33,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:33,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:33,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:33,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:33,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:33,393 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/89eb9f60e9e2423da1cfc49c174f43b6 2024-11-21T21:15:33,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/717540fd17ef4d9fb820b2516c04e2b4 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/717540fd17ef4d9fb820b2516c04e2b4 2024-11-21T21:15:33,406 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/717540fd17ef4d9fb820b2516c04e2b4, entries=200, sequenceid=369, filesize=14.4 K 2024-11-21T21:15:33,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/1ac37caa6e9445fc95f3040dec2b9cd0 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/1ac37caa6e9445fc95f3040dec2b9cd0 2024-11-21T21:15:33,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/1ac37caa6e9445fc95f3040dec2b9cd0, entries=150, sequenceid=369, filesize=12.0 K 2024-11-21T21:15:33,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/89eb9f60e9e2423da1cfc49c174f43b6 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/89eb9f60e9e2423da1cfc49c174f43b6 2024-11-21T21:15:33,419 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-21T21:15:33,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:33,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:33,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:33,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:33,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:33,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:33,422 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/89eb9f60e9e2423da1cfc49c174f43b6, entries=150, sequenceid=369, filesize=12.0 K 2024-11-21T21:15:33,424 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 4a01a47e5e966934386d5c50a408380a in 1271ms, sequenceid=369, compaction requested=true 2024-11-21T21:15:33,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:33,424 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:15:33,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a01a47e5e966934386d5c50a408380a:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:15:33,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:33,424 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:15:33,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a01a47e5e966934386d5c50a408380a:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:15:33,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:33,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a01a47e5e966934386d5c50a408380a:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:15:33,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:15:33,427 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:15:33,427 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:15:33,427 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): 4a01a47e5e966934386d5c50a408380a/A is initiating minor compaction (all files) 2024-11-21T21:15:33,427 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): 4a01a47e5e966934386d5c50a408380a/B is initiating minor compaction (all files) 2024-11-21T21:15:33,427 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a01a47e5e966934386d5c50a408380a/A in TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:33,427 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a01a47e5e966934386d5c50a408380a/B in TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:33,427 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/774676ec24d04102b570ae24617068d6, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/64f2c7edaef0402ea70f77c6fcad0736, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/717540fd17ef4d9fb820b2516c04e2b4] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp, totalSize=39.2 K 2024-11-21T21:15:33,427 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/7281d436b7b04f6993639981e0eb88b3, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/ff1e6dda46f84b15837f8f46c634acd9, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/1ac37caa6e9445fc95f3040dec2b9cd0] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp, totalSize=36.8 K 2024-11-21T21:15:33,427 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 7281d436b7b04f6993639981e0eb88b3, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1732223729632 2024-11-21T21:15:33,427 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 774676ec24d04102b570ae24617068d6, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1732223729632 2024-11-21T21:15:33,428 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting ff1e6dda46f84b15837f8f46c634acd9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732223730305 2024-11-21T21:15:33,428 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64f2c7edaef0402ea70f77c6fcad0736, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732223730305 2024-11-21T21:15:33,428 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ac37caa6e9445fc95f3040dec2b9cd0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732223731504 2024-11-21T21:15:33,428 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 717540fd17ef4d9fb820b2516c04e2b4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732223731504 2024-11-21T21:15:33,435 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a01a47e5e966934386d5c50a408380a#A#compaction#279 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:33,435 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/d2eaa8ec1d7a492aac941c69a9f84e9e is 50, key is test_row_0/A:col10/1732223731504/Put/seqid=0 2024-11-21T21:15:33,438 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a01a47e5e966934386d5c50a408380a#B#compaction#280 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:33,438 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/25bdf7f75fb1430598e7e2c4ed456e64 is 50, key is test_row_0/B:col10/1732223731504/Put/seqid=0 2024-11-21T21:15:33,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742142_1318 (size=13221) 2024-11-21T21:15:33,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742143_1319 (size=13221) 2024-11-21T21:15:33,505 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/25bdf7f75fb1430598e7e2c4ed456e64 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/25bdf7f75fb1430598e7e2c4ed456e64 2024-11-21T21:15:33,516 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a01a47e5e966934386d5c50a408380a/B of 4a01a47e5e966934386d5c50a408380a into 25bdf7f75fb1430598e7e2c4ed456e64(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:33,516 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:33,516 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., storeName=4a01a47e5e966934386d5c50a408380a/B, priority=13, startTime=1732223733424; duration=0sec 2024-11-21T21:15:33,516 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:15:33,516 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a01a47e5e966934386d5c50a408380a:B 2024-11-21T21:15:33,516 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:15:33,519 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:15:33,519 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): 4a01a47e5e966934386d5c50a408380a/C is initiating minor compaction (all files) 2024-11-21T21:15:33,519 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a01a47e5e966934386d5c50a408380a/C in TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:33,519 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/89c3dba9870f4b7689e35474afa5e79b, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/f95ffed44a6047919d1fd1cda00aabc9, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/89eb9f60e9e2423da1cfc49c174f43b6] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp, totalSize=36.8 K 2024-11-21T21:15:33,519 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 89c3dba9870f4b7689e35474afa5e79b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1732223729632 2024-11-21T21:15:33,520 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting f95ffed44a6047919d1fd1cda00aabc9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732223730305 2024-11-21T21:15:33,520 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 89eb9f60e9e2423da1cfc49c174f43b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732223731504 2024-11-21T21:15:33,531 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a01a47e5e966934386d5c50a408380a#C#compaction#281 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:33,532 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/dabaf0c6ca1f4b99ae31125f35782b92 is 50, key is test_row_0/C:col10/1732223731504/Put/seqid=0 2024-11-21T21:15:33,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742144_1320 (size=13221) 2024-11-21T21:15:33,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-21T21:15:33,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:33,573 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 4a01a47e5e966934386d5c50a408380a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T21:15:33,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=A 2024-11-21T21:15:33,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:33,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=B 2024-11-21T21:15:33,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:33,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=C 2024-11-21T21:15:33,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:33,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/734eb414efc64976a33b0d885f061622 is 50, key is test_row_0/A:col10/1732223732172/Put/seqid=0 2024-11-21T21:15:33,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742145_1321 (size=12301) 2024-11-21T21:15:33,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 4a01a47e5e966934386d5c50a408380a 2024-11-21T21:15:33,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:33,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732223733750, max timestamp=1732223733771Num uniques 859; 2024-11-21T21:15:33,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T21:15:33,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732223733750, max timestamp=1732223733771Num uniques 859; 2024-11-21T21:15:33,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732223733750, max timestamp=1732223733771Num uniques 859; 2024-11-21T21:15:33,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T21:15:33,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T21:15:33,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-21T21:15:33,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:33,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223793834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:33,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:33,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:33,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:33,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223793835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:33,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:33,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:33,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:33,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:33,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:33,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223793841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:33,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:33,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:33,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:33,878 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/d2eaa8ec1d7a492aac941c69a9f84e9e as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/d2eaa8ec1d7a492aac941c69a9f84e9e 2024-11-21T21:15:33,884 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a01a47e5e966934386d5c50a408380a/A of 4a01a47e5e966934386d5c50a408380a into d2eaa8ec1d7a492aac941c69a9f84e9e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:33,884 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:33,884 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., storeName=4a01a47e5e966934386d5c50a408380a/A, priority=13, startTime=1732223733424; duration=0sec 2024-11-21T21:15:33,884 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:33,884 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a01a47e5e966934386d5c50a408380a:A 2024-11-21T21:15:33,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:33,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223793950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:33,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:33,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:33,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:33,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223793951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:33,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:33,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:33,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:33,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:33,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:33,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223793951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:33,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:33,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:33,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:33,976 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/dabaf0c6ca1f4b99ae31125f35782b92 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/dabaf0c6ca1f4b99ae31125f35782b92 2024-11-21T21:15:33,981 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a01a47e5e966934386d5c50a408380a/C of 4a01a47e5e966934386d5c50a408380a into dabaf0c6ca1f4b99ae31125f35782b92(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:33,981 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:33,981 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., storeName=4a01a47e5e966934386d5c50a408380a/C, priority=13, startTime=1732223733424; duration=0sec 2024-11-21T21:15:33,981 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:33,981 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a01a47e5e966934386d5c50a408380a:C 2024-11-21T21:15:33,982 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/734eb414efc64976a33b0d885f061622 2024-11-21T21:15:33,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/4c7a10b72d6c4cbda0647934d8491963 is 50, key is test_row_0/B:col10/1732223732172/Put/seqid=0 2024-11-21T21:15:34,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742146_1322 (size=12301) 2024-11-21T21:15:34,001 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/4c7a10b72d6c4cbda0647934d8491963 2024-11-21T21:15:34,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/ac4f37ccdbc344ffbd4a156a7372f664 is 50, key is test_row_0/C:col10/1732223732172/Put/seqid=0 2024-11-21T21:15:34,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742147_1323 (size=12301) 2024-11-21T21:15:34,058 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/ac4f37ccdbc344ffbd4a156a7372f664 2024-11-21T21:15:34,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/734eb414efc64976a33b0d885f061622 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/734eb414efc64976a33b0d885f061622 2024-11-21T21:15:34,071 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/734eb414efc64976a33b0d885f061622, entries=150, sequenceid=380, filesize=12.0 K 2024-11-21T21:15:34,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/4c7a10b72d6c4cbda0647934d8491963 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/4c7a10b72d6c4cbda0647934d8491963 2024-11-21T21:15:34,078 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/4c7a10b72d6c4cbda0647934d8491963, entries=150, sequenceid=380, filesize=12.0 K 2024-11-21T21:15:34,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/ac4f37ccdbc344ffbd4a156a7372f664 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/ac4f37ccdbc344ffbd4a156a7372f664 2024-11-21T21:15:34,084 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/ac4f37ccdbc344ffbd4a156a7372f664, entries=150, sequenceid=380, filesize=12.0 K 2024-11-21T21:15:34,085 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 4a01a47e5e966934386d5c50a408380a in 512ms, sequenceid=380, compaction requested=false 2024-11-21T21:15:34,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:34,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:34,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-21T21:15:34,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-21T21:15:34,093 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-21T21:15:34,093 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4350 sec 2024-11-21T21:15:34,096 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.4420 sec 2024-11-21T21:15:34,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 4a01a47e5e966934386d5c50a408380a 2024-11-21T21:15:34,163 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a01a47e5e966934386d5c50a408380a 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-21T21:15:34,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=A 2024-11-21T21:15:34,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-21T21:15:34,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=B 2024-11-21T21:15:34,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-21T21:15:34,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=C 2024-11-21T21:15:34,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-21T21:15:34,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/e4e8376788794bb1b567cbc7468b597f is 50, key is test_row_0/A:col10/1732223733831/Put/seqid=0 2024-11-21T21:15:34,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742148_1324 (size=14741) 2024-11-21T21:15:34,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:34,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223794171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:34,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:34,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:34,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:34,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:34,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223794172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:34,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:34,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:34,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:34,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:34,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223794177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:34,182 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:34,182 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:34,182 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:34,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:34,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223794280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:34,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:34,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:34,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:34,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:34,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223794280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:34,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:34,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:34,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:34,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:34,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223794290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:34,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:34,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:34,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:34,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:34,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223794330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:34,333 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,41213,1732223653469 for 'test_row_2' in 4a01a47e5e966934386d5c50a408380a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:15:34,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:34,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:34,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:34,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:34,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223794340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:34,345 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,41213,1732223653469 for 'test_row_2' in 4a01a47e5e966934386d5c50a408380a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:15:34,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:34,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:34,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:34,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:34,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223794490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:34,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:34,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223794490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:34,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:34,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:34,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:34,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:34,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:34,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:34,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:34,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223794500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:34,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:34,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:34,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:34,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/e4e8376788794bb1b567cbc7468b597f 2024-11-21T21:15:34,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/bf823efb7fcb4ef88cfdac6715eec433 is 50, key is test_row_0/B:col10/1732223733831/Put/seqid=0 2024-11-21T21:15:34,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742149_1325 (size=12301) 2024-11-21T21:15:34,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:34,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223794801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:34,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:34,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:34,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:34,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:34,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223794801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:34,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:34,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:34,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:34,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-21T21:15:34,810 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:15:34,811 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:15:34,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-21T21:15:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-21T21:15:34,813 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:15:34,813 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:15:34,813 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:15:34,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:34,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223794810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:34,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:34,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:34,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:34,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-21T21:15:34,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-21T21:15:34,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:34,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:34,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:34,965 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:34,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:34,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:34,984 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/bf823efb7fcb4ef88cfdac6715eec433 2024-11-21T21:15:34,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/d0c5290ee4554826b9f2dc2efc6cdfc0 is 50, key is test_row_0/C:col10/1732223733831/Put/seqid=0 2024-11-21T21:15:35,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742150_1326 (size=12301) 2024-11-21T21:15:35,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-21T21:15:35,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:35,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:35,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:35,118 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:35,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:35,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:35,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-21T21:15:35,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-21T21:15:35,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:35,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:35,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:35,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:35,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:35,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:35,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:35,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223795310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:35,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:35,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:35,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:35,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:35,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223795320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:35,324 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:35,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:35,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:35,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:35,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223795320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:35,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:35,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:35,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:35,402 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/d0c5290ee4554826b9f2dc2efc6cdfc0 2024-11-21T21:15:35,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/e4e8376788794bb1b567cbc7468b597f as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/e4e8376788794bb1b567cbc7468b597f 2024-11-21T21:15:35,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/e4e8376788794bb1b567cbc7468b597f, entries=200, sequenceid=420, filesize=14.4 K 2024-11-21T21:15:35,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/bf823efb7fcb4ef88cfdac6715eec433 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/bf823efb7fcb4ef88cfdac6715eec433 2024-11-21T21:15:35,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/bf823efb7fcb4ef88cfdac6715eec433, entries=150, sequenceid=420, filesize=12.0 K 2024-11-21T21:15:35,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/d0c5290ee4554826b9f2dc2efc6cdfc0 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/d0c5290ee4554826b9f2dc2efc6cdfc0 2024-11-21T21:15:35,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/d0c5290ee4554826b9f2dc2efc6cdfc0, entries=150, sequenceid=420, filesize=12.0 K 2024-11-21T21:15:35,419 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 4a01a47e5e966934386d5c50a408380a in 1256ms, sequenceid=420, compaction requested=true 2024-11-21T21:15:35,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:35,419 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:15:35,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a01a47e5e966934386d5c50a408380a:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:15:35,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:35,420 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:15:35,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a01a47e5e966934386d5c50a408380a:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:15:35,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:35,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a01a47e5e966934386d5c50a408380a:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:15:35,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:15:35,420 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40263 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:15:35,420 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:15:35,420 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): 4a01a47e5e966934386d5c50a408380a/A is initiating minor compaction (all files) 2024-11-21T21:15:35,421 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): 4a01a47e5e966934386d5c50a408380a/B is initiating minor compaction (all files) 2024-11-21T21:15:35,421 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a01a47e5e966934386d5c50a408380a/A in TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:35,421 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a01a47e5e966934386d5c50a408380a/B in TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:35,421 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/d2eaa8ec1d7a492aac941c69a9f84e9e, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/734eb414efc64976a33b0d885f061622, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/e4e8376788794bb1b567cbc7468b597f] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp, totalSize=39.3 K 2024-11-21T21:15:35,421 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/25bdf7f75fb1430598e7e2c4ed456e64, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/4c7a10b72d6c4cbda0647934d8491963, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/bf823efb7fcb4ef88cfdac6715eec433] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp, totalSize=36.9 K 2024-11-21T21:15:35,421 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2eaa8ec1d7a492aac941c69a9f84e9e, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732223731504 2024-11-21T21:15:35,421 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 25bdf7f75fb1430598e7e2c4ed456e64, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732223731504 2024-11-21T21:15:35,421 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c7a10b72d6c4cbda0647934d8491963, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1732223732159 2024-11-21T21:15:35,421 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 734eb414efc64976a33b0d885f061622, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1732223732159 2024-11-21T21:15:35,422 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting bf823efb7fcb4ef88cfdac6715eec433, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1732223733831 2024-11-21T21:15:35,422 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4e8376788794bb1b567cbc7468b597f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1732223733823 2024-11-21T21:15:35,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-21T21:15:35,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:35,423 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 4a01a47e5e966934386d5c50a408380a 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-21T21:15:35,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=A 2024-11-21T21:15:35,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:35,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=B 2024-11-21T21:15:35,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:35,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=C 2024-11-21T21:15:35,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:35,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/a40f283cfdf248df8b6c9fe6cbb65adb is 50, key is test_row_0/A:col10/1732223734170/Put/seqid=0 2024-11-21T21:15:35,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-21T21:15:35,442 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a01a47e5e966934386d5c50a408380a#B#compaction#289 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:35,442 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a01a47e5e966934386d5c50a408380a#A#compaction#290 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:35,443 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/59c07620829e481c9f1d7ac8d0aa9973 is 50, key is test_row_0/B:col10/1732223733831/Put/seqid=0 2024-11-21T21:15:35,443 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/7fb184da73f940bcbfc75c1f232a6137 is 50, key is test_row_0/A:col10/1732223733831/Put/seqid=0 2024-11-21T21:15:35,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742151_1327 (size=12301) 2024-11-21T21:15:35,456 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/a40f283cfdf248df8b6c9fe6cbb65adb 2024-11-21T21:15:35,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742152_1328 (size=13323) 2024-11-21T21:15:35,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/1513d410f27d4b5e98a3d8824eee261d is 50, key is test_row_0/B:col10/1732223734170/Put/seqid=0 2024-11-21T21:15:35,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742153_1329 (size=13323) 2024-11-21T21:15:35,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742154_1330 (size=12301) 2024-11-21T21:15:35,514 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/1513d410f27d4b5e98a3d8824eee261d 2024-11-21T21:15:35,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/fda8dab5b805458fb9d0e61fe1aa33f3 is 50, key is test_row_0/C:col10/1732223734170/Put/seqid=0 2024-11-21T21:15:35,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742155_1331 (size=12301) 2024-11-21T21:15:35,526 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/fda8dab5b805458fb9d0e61fe1aa33f3 2024-11-21T21:15:35,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/a40f283cfdf248df8b6c9fe6cbb65adb as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/a40f283cfdf248df8b6c9fe6cbb65adb 2024-11-21T21:15:35,535 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/a40f283cfdf248df8b6c9fe6cbb65adb, entries=150, sequenceid=429, filesize=12.0 K 2024-11-21T21:15:35,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/1513d410f27d4b5e98a3d8824eee261d as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/1513d410f27d4b5e98a3d8824eee261d 2024-11-21T21:15:35,540 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/1513d410f27d4b5e98a3d8824eee261d, entries=150, sequenceid=429, filesize=12.0 K 2024-11-21T21:15:35,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/fda8dab5b805458fb9d0e61fe1aa33f3 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/fda8dab5b805458fb9d0e61fe1aa33f3 2024-11-21T21:15:35,545 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/fda8dab5b805458fb9d0e61fe1aa33f3, entries=150, sequenceid=429, filesize=12.0 K 2024-11-21T21:15:35,545 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=0 B/0 for 4a01a47e5e966934386d5c50a408380a in 122ms, sequenceid=429, compaction requested=true 2024-11-21T21:15:35,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:35,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:35,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-21T21:15:35,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-21T21:15:35,549 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-21T21:15:35,549 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 734 msec 2024-11-21T21:15:35,552 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 740 msec 2024-11-21T21:15:35,876 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/59c07620829e481c9f1d7ac8d0aa9973 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/59c07620829e481c9f1d7ac8d0aa9973 2024-11-21T21:15:35,884 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a01a47e5e966934386d5c50a408380a/B of 4a01a47e5e966934386d5c50a408380a into 59c07620829e481c9f1d7ac8d0aa9973(size=13.0 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:35,885 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:35,885 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., storeName=4a01a47e5e966934386d5c50a408380a/B, priority=13, startTime=1732223735420; duration=0sec 2024-11-21T21:15:35,885 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:15:35,885 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a01a47e5e966934386d5c50a408380a:B 2024-11-21T21:15:35,885 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:15:35,886 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:15:35,886 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): 4a01a47e5e966934386d5c50a408380a/C is initiating minor compaction (all files) 2024-11-21T21:15:35,886 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a01a47e5e966934386d5c50a408380a/C in TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:35,886 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/dabaf0c6ca1f4b99ae31125f35782b92, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/ac4f37ccdbc344ffbd4a156a7372f664, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/d0c5290ee4554826b9f2dc2efc6cdfc0, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/fda8dab5b805458fb9d0e61fe1aa33f3] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp, totalSize=48.9 K 2024-11-21T21:15:35,886 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting dabaf0c6ca1f4b99ae31125f35782b92, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732223731504 2024-11-21T21:15:35,887 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting ac4f37ccdbc344ffbd4a156a7372f664, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1732223732159 2024-11-21T21:15:35,887 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting d0c5290ee4554826b9f2dc2efc6cdfc0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1732223733831 2024-11-21T21:15:35,887 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting fda8dab5b805458fb9d0e61fe1aa33f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1732223734165 2024-11-21T21:15:35,896 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a01a47e5e966934386d5c50a408380a#C#compaction#293 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:35,897 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/5fabd25464fd495a9e02b16f51ab78d3 is 50, key is test_row_0/C:col10/1732223734170/Put/seqid=0 2024-11-21T21:15:35,897 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/7fb184da73f940bcbfc75c1f232a6137 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/7fb184da73f940bcbfc75c1f232a6137 2024-11-21T21:15:35,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742156_1332 (size=13357) 2024-11-21T21:15:35,903 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a01a47e5e966934386d5c50a408380a/A of 4a01a47e5e966934386d5c50a408380a into 7fb184da73f940bcbfc75c1f232a6137(size=13.0 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:35,903 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:35,903 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., storeName=4a01a47e5e966934386d5c50a408380a/A, priority=13, startTime=1732223735419; duration=0sec 2024-11-21T21:15:35,903 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:35,903 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a01a47e5e966934386d5c50a408380a:A 2024-11-21T21:15:35,908 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/5fabd25464fd495a9e02b16f51ab78d3 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/5fabd25464fd495a9e02b16f51ab78d3 2024-11-21T21:15:35,912 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4a01a47e5e966934386d5c50a408380a/C of 4a01a47e5e966934386d5c50a408380a into 5fabd25464fd495a9e02b16f51ab78d3(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:35,912 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:35,912 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., storeName=4a01a47e5e966934386d5c50a408380a/C, priority=12, startTime=1732223735420; duration=0sec 2024-11-21T21:15:35,912 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:35,913 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a01a47e5e966934386d5c50a408380a:C 2024-11-21T21:15:35,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-21T21:15:35,950 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:15:35,951 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:15:35,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-21T21:15:35,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-21T21:15:35,952 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:15:35,953 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:15:35,953 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:15:36,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-21T21:15:36,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-21T21:15:36,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:36,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:36,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:36,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-21T21:15:36,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-21T21:15:36,112 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-21T21:15:36,112 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 157 msec 2024-11-21T21:15:36,114 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 162 msec 2024-11-21T21:15:36,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-21T21:15:36,271 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:15:36,271 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:15:36,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees 2024-11-21T21:15:36,273 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:15:36,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-21T21:15:36,274 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:15:36,274 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:15:36,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 4a01a47e5e966934386d5c50a408380a 2024-11-21T21:15:36,360 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a01a47e5e966934386d5c50a408380a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T21:15:36,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=A 2024-11-21T21:15:36,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:36,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=B 2024-11-21T21:15:36,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:36,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=C 2024-11-21T21:15:36,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:36,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/39ab429e6e5e4545845702529f862b28 is 50, key is test_row_0/A:col10/1732223736359/Put/seqid=0 2024-11-21T21:15:36,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742157_1333 (size=12301) 2024-11-21T21:15:36,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-21T21:15:36,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:36,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223796377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:36,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:36,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:36,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:36,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732223736376, max timestamp=1732223736382Num uniques 859; 2024-11-21T21:15:36,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732223736376, max timestamp=1732223736382Num uniques 859; 2024-11-21T21:15:36,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732223736376, max timestamp=1732223736382Num uniques 859; 2024-11-21T21:15:36,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T21:15:36,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T21:15:36,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T21:15:36,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:36,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223796408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:36,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732223736402, max timestamp=1732223736411Num uniques 1018; 2024-11-21T21:15:36,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732223736402, max timestamp=1732223736411Num uniques 1018; 2024-11-21T21:15:36,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-21T21:15:36,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-21T21:15:36,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732223736402, max timestamp=1732223736411Num uniques 1018; 2024-11-21T21:15:36,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-21T21:15:36,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:36,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:36,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:36,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-21T21:15:36,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:36,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:36,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:36,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:36,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:36,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:36,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:36,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223796455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:36,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:36,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:36,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:36,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:36,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223796457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:36,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:36,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:36,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:36,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223796455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:36,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:36,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:36,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:36,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:36,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:36,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223796490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:36,494 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:36,494 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:36,494 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:36,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:36,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223796530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:36,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:36,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:36,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:36,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:36,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223796571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:36,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:36,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:36,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:36,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:36,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223796571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:36,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:36,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:36,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:36,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:36,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223796571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:36,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:36,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:36,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:36,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-21T21:15:36,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:36,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:36,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:36,586 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:36,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:36,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:36,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-21T21:15:36,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:36,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223796700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:36,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:36,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:36,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:36,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-21T21:15:36,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:36,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:36,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:36,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:36,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:36,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:36,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:36,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223796740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:36,744 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:36,744 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:36,744 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:36,769 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/39ab429e6e5e4545845702529f862b28 2024-11-21T21:15:36,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/f81bb18b1f40493aa7a8f23ec94582f8 is 50, key is test_row_0/B:col10/1732223736359/Put/seqid=0 2024-11-21T21:15:36,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742158_1334 (size=12301) 2024-11-21T21:15:36,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:36,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223796781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:36,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:36,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:36,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:36,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223796781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:36,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:36,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:36,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:36,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:36,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:36,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223796781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:36,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:36,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:36,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:36,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-21T21:15:36,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:36,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:36,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:36,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:36,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:36,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:36,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-21T21:15:37,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:37,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223797010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:37,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:37,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:37,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:37,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-21T21:15:37,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:37,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:37,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:37,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:37,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:37,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:37,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:37,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223797050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:37,056 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:37,056 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:37,056 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:37,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:37,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223797090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:37,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:37,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:37,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:37,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223797090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:37,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:37,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:37,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:37,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:37,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:37,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223797090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:37,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:37,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:37,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:37,179 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/f81bb18b1f40493aa7a8f23ec94582f8 2024-11-21T21:15:37,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/44b674a43daf4f33b69d70b5b07fa533 is 50, key is test_row_0/C:col10/1732223736359/Put/seqid=0 2024-11-21T21:15:37,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742159_1335 (size=12301) 2024-11-21T21:15:37,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-21T21:15:37,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:37,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:37,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:37,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:37,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:37,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:37,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-21T21:15:37,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:37,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:37,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:37,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:37,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:37,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:37,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-21T21:15:37,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-21T21:15:37,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:37,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. as already flushing 2024-11-21T21:15:37,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:37,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:37,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:37,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:37,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:37,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223797520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:37,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:37,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:37,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:37,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:37,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223797560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:37,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:37,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:37,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:37,588 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/44b674a43daf4f33b69d70b5b07fa533 2024-11-21T21:15:37,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/39ab429e6e5e4545845702529f862b28 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/39ab429e6e5e4545845702529f862b28 2024-11-21T21:15:37,599 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/39ab429e6e5e4545845702529f862b28, entries=150, sequenceid=443, filesize=12.0 K 2024-11-21T21:15:37,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/f81bb18b1f40493aa7a8f23ec94582f8 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/f81bb18b1f40493aa7a8f23ec94582f8 2024-11-21T21:15:37,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/f81bb18b1f40493aa7a8f23ec94582f8, entries=150, sequenceid=443, filesize=12.0 K 2024-11-21T21:15:37,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:37,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223797600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:37,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:37,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223797600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:37,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:37,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:37,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:37,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:37,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:37,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/44b674a43daf4f33b69d70b5b07fa533 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/44b674a43daf4f33b69d70b5b07fa533 2024-11-21T21:15:37,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:37,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:37,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223797600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:37,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:37,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:37,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:37,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/44b674a43daf4f33b69d70b5b07fa533, entries=150, sequenceid=443, filesize=12.0 K 2024-11-21T21:15:37,612 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 4a01a47e5e966934386d5c50a408380a in 1252ms, sequenceid=443, compaction requested=true 2024-11-21T21:15:37,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:37,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a01a47e5e966934386d5c50a408380a:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:15:37,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:37,613 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:15:37,613 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:15:37,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a01a47e5e966934386d5c50a408380a:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:15:37,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:37,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a01a47e5e966934386d5c50a408380a:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:15:37,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:15:37,614 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:15:37,614 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:15:37,614 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): 4a01a47e5e966934386d5c50a408380a/B is initiating minor compaction (all files) 2024-11-21T21:15:37,614 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): 4a01a47e5e966934386d5c50a408380a/A is initiating minor compaction (all files) 2024-11-21T21:15:37,614 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a01a47e5e966934386d5c50a408380a/A in TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:37,614 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a01a47e5e966934386d5c50a408380a/B in TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:37,614 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/59c07620829e481c9f1d7ac8d0aa9973, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/1513d410f27d4b5e98a3d8824eee261d, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/f81bb18b1f40493aa7a8f23ec94582f8] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp, totalSize=37.0 K 2024-11-21T21:15:37,614 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/7fb184da73f940bcbfc75c1f232a6137, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/a40f283cfdf248df8b6c9fe6cbb65adb, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/39ab429e6e5e4545845702529f862b28] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp, totalSize=37.0 K 2024-11-21T21:15:37,614 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fb184da73f940bcbfc75c1f232a6137, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1732223733831 2024-11-21T21:15:37,614 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 59c07620829e481c9f1d7ac8d0aa9973, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1732223733831 2024-11-21T21:15:37,615 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 1513d410f27d4b5e98a3d8824eee261d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1732223734165 2024-11-21T21:15:37,615 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting f81bb18b1f40493aa7a8f23ec94582f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1732223736343 2024-11-21T21:15:37,615 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting a40f283cfdf248df8b6c9fe6cbb65adb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1732223734165 2024-11-21T21:15:37,616 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39ab429e6e5e4545845702529f862b28, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1732223736343 2024-11-21T21:15:37,628 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a01a47e5e966934386d5c50a408380a#A#compaction#297 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:37,629 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/2f5c3b635cdd468bbd3b23cd6dc72b9b is 50, key is test_row_0/A:col10/1732223736359/Put/seqid=0 2024-11-21T21:15:37,642 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a01a47e5e966934386d5c50a408380a#B#compaction#298 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:37,643 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/8fc06e51021640309a52925ad1a4e56a is 50, key is test_row_0/B:col10/1732223736359/Put/seqid=0 2024-11-21T21:15:37,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742160_1336 (size=13425) 2024-11-21T21:15:37,651 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/2f5c3b635cdd468bbd3b23cd6dc72b9b as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/2f5c3b635cdd468bbd3b23cd6dc72b9b 2024-11-21T21:15:37,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-21T21:15:37,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:37,655 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2902): Flushing 4a01a47e5e966934386d5c50a408380a 3/3 column families, dataSize=134.18 KB heapSize=374.77 KB 2024-11-21T21:15:37,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=A 2024-11-21T21:15:37,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:15:37,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=B 2024-11-21T21:15:37,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:15:37,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=C 2024-11-21T21:15:37,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:15:37,657 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a01a47e5e966934386d5c50a408380a/A of 4a01a47e5e966934386d5c50a408380a into 2f5c3b635cdd468bbd3b23cd6dc72b9b(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:37,657 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:37,657 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., storeName=4a01a47e5e966934386d5c50a408380a/A, priority=13, startTime=1732223737613; duration=0sec 2024-11-21T21:15:37,657 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:15:37,657 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a01a47e5e966934386d5c50a408380a:A 2024-11-21T21:15:37,657 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-21T21:15:37,659 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-21T21:15:37,659 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-21T21:15:37,659 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. because compaction request was cancelled 2024-11-21T21:15:37,659 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a01a47e5e966934386d5c50a408380a:C 2024-11-21T21:15:37,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742161_1337 (size=13425) 2024-11-21T21:15:37,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/f4d0b3b2d9b4489fbf18d79ccf6c9950 is 50, key is test_row_0/A:col10/1732223736442/Put/seqid=0 2024-11-21T21:15:37,666 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/8fc06e51021640309a52925ad1a4e56a as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/8fc06e51021640309a52925ad1a4e56a 2024-11-21T21:15:37,672 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a01a47e5e966934386d5c50a408380a/B of 4a01a47e5e966934386d5c50a408380a into 8fc06e51021640309a52925ad1a4e56a(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:37,672 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:37,672 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., storeName=4a01a47e5e966934386d5c50a408380a/B, priority=13, startTime=1732223737613; duration=0sec 2024-11-21T21:15:37,672 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:37,672 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a01a47e5e966934386d5c50a408380a:B 2024-11-21T21:15:37,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742162_1338 (size=12301) 2024-11-21T21:15:38,077 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=495 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/f4d0b3b2d9b4489fbf18d79ccf6c9950 2024-11-21T21:15:38,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/d96ba76cbcc541b8b612ae86e85bc18f is 50, key is test_row_0/B:col10/1732223736442/Put/seqid=0 2024-11-21T21:15:38,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742163_1339 (size=12301) 2024-11-21T21:15:38,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-21T21:15:38,488 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=495 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/d96ba76cbcc541b8b612ae86e85bc18f 2024-11-21T21:15:38,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/e97f9d7dfab5490a9a540a27af7e03ba is 50, key is test_row_0/C:col10/1732223736442/Put/seqid=0 2024-11-21T21:15:38,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742164_1340 (size=12301) 2024-11-21T21:15:38,510 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=495 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/e97f9d7dfab5490a9a540a27af7e03ba 2024-11-21T21:15:38,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/f4d0b3b2d9b4489fbf18d79ccf6c9950 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/f4d0b3b2d9b4489fbf18d79ccf6c9950 2024-11-21T21:15:38,517 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/f4d0b3b2d9b4489fbf18d79ccf6c9950, entries=150, sequenceid=495, filesize=12.0 K 2024-11-21T21:15:38,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/d96ba76cbcc541b8b612ae86e85bc18f as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/d96ba76cbcc541b8b612ae86e85bc18f 2024-11-21T21:15:38,521 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/d96ba76cbcc541b8b612ae86e85bc18f, entries=150, sequenceid=495, filesize=12.0 K 2024-11-21T21:15:38,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/e97f9d7dfab5490a9a540a27af7e03ba as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/e97f9d7dfab5490a9a540a27af7e03ba 2024-11-21T21:15:38,526 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/e97f9d7dfab5490a9a540a27af7e03ba, entries=150, sequenceid=495, filesize=12.0 K 2024-11-21T21:15:38,527 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~374.72 KB/383712, currentSize=0 B/0 for 4a01a47e5e966934386d5c50a408380a in 871ms, sequenceid=495, compaction requested=true 2024-11-21T21:15:38,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2603): Flush status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:38,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:38,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=87 2024-11-21T21:15:38,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=87 2024-11-21T21:15:38,530 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-21T21:15:38,530 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2540 sec 2024-11-21T21:15:38,531 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees in 2.2590 sec 2024-11-21T21:15:38,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 4a01a47e5e966934386d5c50a408380a 2024-11-21T21:15:38,548 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a01a47e5e966934386d5c50a408380a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T21:15:38,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=A 2024-11-21T21:15:38,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:38,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=B 2024-11-21T21:15:38,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:38,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=C 2024-11-21T21:15:38,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:38,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/11872f13a5134943b12a01bf05a7c015 is 50, key is test_row_0/A:col10/1732223738544/Put/seqid=0 2024-11-21T21:15:38,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742165_1341 (size=14741) 2024-11-21T21:15:38,604 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732223738584, max timestamp=1732223738602Num uniques 859; 2024-11-21T21:15:38,604 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T21:15:38,605 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732223738584, max timestamp=1732223738602Num uniques 859; 2024-11-21T21:15:38,605 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T21:15:38,605 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732223738584, max timestamp=1732223738602Num uniques 859; 2024-11-21T21:15:38,605 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T21:15:38,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:38,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223798626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:38,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:38,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:38,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:38,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:38,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223798627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:38,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:38,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:38,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:38,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:38,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223798627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:38,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:38,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:38,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:38,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:38,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223798628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:38,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:38,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:38,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:38,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:38,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223798628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:38,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:38,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:38,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:38,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:38,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223798740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:38,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:38,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:38,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223798740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:38,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:38,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:38,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:38,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:38,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:38,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:38,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223798741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:38,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:38,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:38,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223798741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:38,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:38,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:38,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:38,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:38,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:38,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:38,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223798741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:38,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:38,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:38,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:38,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:38,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:38,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223798951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:38,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223798951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:38,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:38,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:38,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223798951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:38,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223798951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:38,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:38,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:38,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223798951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:38,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:38,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:38,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:38,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:38,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:38,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:38,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:38,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:38,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:38,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:38,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:38,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:38,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:38,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:38,960 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/11872f13a5134943b12a01bf05a7c015 2024-11-21T21:15:38,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/5203815b13444d2580a961544401af9f is 50, key is test_row_0/B:col10/1732223738544/Put/seqid=0 2024-11-21T21:15:38,980 INFO [Thread-1117 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1117. 2024-11-21T21:15:38,980 DEBUG [Thread-1117 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:15:38,980 DEBUG [Thread-1117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:38,980 DEBUG [Thread-1117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:38,980 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:15:38,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742166_1342 (size=12301) 2024-11-21T21:15:38,982 INFO [Thread-1123 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1123. 2024-11-21T21:15:38,982 DEBUG [Thread-1123 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:15:38,982 DEBUG [Thread-1123 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:38,982 DEBUG [Thread-1123 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:38,982 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:15:38,983 INFO [Thread-1121 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1121. 2024-11-21T21:15:38,983 DEBUG [Thread-1121 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:15:38,983 DEBUG [Thread-1121 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:38,984 DEBUG [Thread-1121 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:38,984 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:15:38,984 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/5203815b13444d2580a961544401af9f 2024-11-21T21:15:38,987 INFO [Thread-1115 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1115. 2024-11-21T21:15:38,987 DEBUG [Thread-1115 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:15:38,987 DEBUG [Thread-1115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:38,987 DEBUG [Thread-1115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:38,987 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:15:38,988 INFO [Thread-1119 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1119. 2024-11-21T21:15:38,988 DEBUG [Thread-1119 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:15:38,988 DEBUG [Thread-1119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:38,988 DEBUG [Thread-1119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:38,988 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:15:38,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/4e6b1d5755694888a719f42a3e230a80 is 50, key is test_row_0/C:col10/1732223738544/Put/seqid=0 2024-11-21T21:15:38,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742167_1343 (size=12301) 2024-11-21T21:15:39,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:39,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53168 deadline: 1732223799261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:39,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:39,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53220 deadline: 1732223799261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:39,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:39,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:39,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:39,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:39,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:39,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:39,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:39,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:39,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53170 deadline: 1732223799270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:39,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:39,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53234 deadline: 1732223799271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:39,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53198 deadline: 1732223799271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:39,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:39,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:39,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:39,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:39,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:39,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:39,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:39,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:39,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:15:39,398 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/4e6b1d5755694888a719f42a3e230a80 2024-11-21T21:15:39,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/11872f13a5134943b12a01bf05a7c015 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/11872f13a5134943b12a01bf05a7c015 2024-11-21T21:15:39,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/11872f13a5134943b12a01bf05a7c015, entries=200, sequenceid=507, filesize=14.4 K 2024-11-21T21:15:39,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/5203815b13444d2580a961544401af9f as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/5203815b13444d2580a961544401af9f 2024-11-21T21:15:39,420 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/5203815b13444d2580a961544401af9f, entries=150, sequenceid=507, filesize=12.0 K 2024-11-21T21:15:39,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/4e6b1d5755694888a719f42a3e230a80 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/4e6b1d5755694888a719f42a3e230a80 2024-11-21T21:15:39,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/4e6b1d5755694888a719f42a3e230a80, entries=150, sequenceid=507, filesize=12.0 K 2024-11-21T21:15:39,431 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 4a01a47e5e966934386d5c50a408380a in 883ms, sequenceid=507, compaction requested=true 2024-11-21T21:15:39,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:39,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a01a47e5e966934386d5c50a408380a:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:15:39,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:39,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a01a47e5e966934386d5c50a408380a:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:15:39,431 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:15:39,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:39,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a01a47e5e966934386d5c50a408380a:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:15:39,431 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:15:39,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:15:39,432 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:15:39,432 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40467 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:15:39,432 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): 4a01a47e5e966934386d5c50a408380a/B is initiating minor compaction (all files) 2024-11-21T21:15:39,432 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): 4a01a47e5e966934386d5c50a408380a/A is initiating minor compaction (all files) 2024-11-21T21:15:39,432 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a01a47e5e966934386d5c50a408380a/A in TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:39,432 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a01a47e5e966934386d5c50a408380a/B in TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:39,432 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/2f5c3b635cdd468bbd3b23cd6dc72b9b, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/f4d0b3b2d9b4489fbf18d79ccf6c9950, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/11872f13a5134943b12a01bf05a7c015] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp, totalSize=39.5 K 2024-11-21T21:15:39,432 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/8fc06e51021640309a52925ad1a4e56a, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/d96ba76cbcc541b8b612ae86e85bc18f, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/5203815b13444d2580a961544401af9f] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp, totalSize=37.1 K 2024-11-21T21:15:39,432 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f5c3b635cdd468bbd3b23cd6dc72b9b, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1732223736343 2024-11-21T21:15:39,432 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fc06e51021640309a52925ad1a4e56a, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1732223736343 2024-11-21T21:15:39,432 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting d96ba76cbcc541b8b612ae86e85bc18f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=495, earliestPutTs=1732223736442 2024-11-21T21:15:39,432 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4d0b3b2d9b4489fbf18d79ccf6c9950, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=495, earliestPutTs=1732223736442 2024-11-21T21:15:39,433 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 5203815b13444d2580a961544401af9f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=507, earliestPutTs=1732223738538 2024-11-21T21:15:39,433 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11872f13a5134943b12a01bf05a7c015, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=507, earliestPutTs=1732223738535 2024-11-21T21:15:39,439 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a01a47e5e966934386d5c50a408380a#A#compaction#306 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:39,439 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a01a47e5e966934386d5c50a408380a#B#compaction#305 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:39,439 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/e25f7a26e3c54d2c99a5bab456b688c9 is 50, key is test_row_0/A:col10/1732223738544/Put/seqid=0 2024-11-21T21:15:39,439 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/a284e07630594afc9b07d2d6d83235d9 is 50, key is test_row_0/B:col10/1732223738544/Put/seqid=0 2024-11-21T21:15:39,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742169_1345 (size=13527) 2024-11-21T21:15:39,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742168_1344 (size=13527) 2024-11-21T21:15:39,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 4a01a47e5e966934386d5c50a408380a 2024-11-21T21:15:39,772 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a01a47e5e966934386d5c50a408380a 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-21T21:15:39,772 INFO [Thread-1106 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1106. 2024-11-21T21:15:39,772 INFO [Thread-1104 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1104. 2024-11-21T21:15:39,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=A 2024-11-21T21:15:39,773 DEBUG [Thread-1104 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:15:39,773 DEBUG [Thread-1106 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:15:39,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-21T21:15:39,773 DEBUG [Thread-1104 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:39,773 DEBUG [Thread-1106 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:39,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=B 2024-11-21T21:15:39,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-21T21:15:39,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=C 2024-11-21T21:15:39,773 DEBUG [Thread-1104 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:39,773 DEBUG [Thread-1106 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:39,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-21T21:15:39,773 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:15:39,773 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:15:39,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/275c6430018a4055bb4349174cf90acd is 50, key is test_row_0/A:col10/1732223738624/Put/seqid=0 2024-11-21T21:15:39,783 INFO [Thread-1112 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1112. 2024-11-21T21:15:39,783 INFO [Thread-1108 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1108. 2024-11-21T21:15:39,783 INFO [Thread-1110 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1110. 2024-11-21T21:15:39,783 DEBUG [Thread-1112 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:15:39,783 DEBUG [Thread-1108 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:15:39,783 DEBUG [Thread-1110 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:15:39,783 DEBUG [Thread-1112 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:39,783 DEBUG [Thread-1108 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:39,783 DEBUG [Thread-1110 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:39,783 DEBUG [Thread-1110 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:39,783 DEBUG [Thread-1108 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:39,783 DEBUG [Thread-1112 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:39,783 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:15:39,783 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:15:39,783 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:15:39,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742170_1346 (size=12301) 2024-11-21T21:15:39,784 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=545 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/275c6430018a4055bb4349174cf90acd 2024-11-21T21:15:39,789 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/771c09ff092e421b8b51c7bc415d7354 is 50, key is test_row_0/B:col10/1732223738624/Put/seqid=0 2024-11-21T21:15:39,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742171_1347 (size=12301) 2024-11-21T21:15:39,851 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/e25f7a26e3c54d2c99a5bab456b688c9 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/e25f7a26e3c54d2c99a5bab456b688c9 2024-11-21T21:15:39,851 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/a284e07630594afc9b07d2d6d83235d9 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/a284e07630594afc9b07d2d6d83235d9 2024-11-21T21:15:39,857 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a01a47e5e966934386d5c50a408380a/A of 4a01a47e5e966934386d5c50a408380a into e25f7a26e3c54d2c99a5bab456b688c9(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:39,857 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a01a47e5e966934386d5c50a408380a/B of 4a01a47e5e966934386d5c50a408380a into a284e07630594afc9b07d2d6d83235d9(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:39,857 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:39,857 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:39,857 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., storeName=4a01a47e5e966934386d5c50a408380a/A, priority=13, startTime=1732223739431; duration=0sec 2024-11-21T21:15:39,857 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., storeName=4a01a47e5e966934386d5c50a408380a/B, priority=13, startTime=1732223739431; duration=0sec 2024-11-21T21:15:39,857 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:15:39,857 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:15:39,858 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a01a47e5e966934386d5c50a408380a:B 2024-11-21T21:15:39,858 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a01a47e5e966934386d5c50a408380a:A 2024-11-21T21:15:39,858 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:15:39,859 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50260 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:15:39,859 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): 4a01a47e5e966934386d5c50a408380a/C is initiating minor compaction (all files) 2024-11-21T21:15:39,859 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a01a47e5e966934386d5c50a408380a/C in TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:39,859 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/5fabd25464fd495a9e02b16f51ab78d3, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/44b674a43daf4f33b69d70b5b07fa533, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/e97f9d7dfab5490a9a540a27af7e03ba, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/4e6b1d5755694888a719f42a3e230a80] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp, totalSize=49.1 K 2024-11-21T21:15:39,860 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fabd25464fd495a9e02b16f51ab78d3, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1732223734165 2024-11-21T21:15:39,860 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 44b674a43daf4f33b69d70b5b07fa533, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1732223736343 2024-11-21T21:15:39,860 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting e97f9d7dfab5490a9a540a27af7e03ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=495, earliestPutTs=1732223736442 2024-11-21T21:15:39,860 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e6b1d5755694888a719f42a3e230a80, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=507, earliestPutTs=1732223738538 2024-11-21T21:15:39,867 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a01a47e5e966934386d5c50a408380a#C#compaction#309 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:39,867 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/b2ca71a40904495ab7bcb7f59ad3ba15 is 50, key is test_row_0/C:col10/1732223738544/Put/seqid=0 2024-11-21T21:15:39,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742172_1348 (size=13493) 2024-11-21T21:15:40,193 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=545 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/771c09ff092e421b8b51c7bc415d7354 2024-11-21T21:15:40,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/0d72e94cefaa43a9b2860828c5dce153 is 50, key is test_row_0/C:col10/1732223738624/Put/seqid=0 2024-11-21T21:15:40,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742173_1349 (size=12301) 2024-11-21T21:15:40,276 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/b2ca71a40904495ab7bcb7f59ad3ba15 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/b2ca71a40904495ab7bcb7f59ad3ba15 2024-11-21T21:15:40,281 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4a01a47e5e966934386d5c50a408380a/C of 4a01a47e5e966934386d5c50a408380a into b2ca71a40904495ab7bcb7f59ad3ba15(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:40,281 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:40,281 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., storeName=4a01a47e5e966934386d5c50a408380a/C, priority=12, startTime=1732223739431; duration=0sec 2024-11-21T21:15:40,281 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:40,281 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a01a47e5e966934386d5c50a408380a:C 2024-11-21T21:15:40,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-21T21:15:40,431 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:15:40,431 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T21:15:40,431 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 86 2024-11-21T21:15:40,431 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-11-21T21:15:40,431 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-21T21:15:40,431 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 94 2024-11-21T21:15:40,431 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 119 2024-11-21T21:15:40,431 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T21:15:40,431 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2897 2024-11-21T21:15:40,431 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2885 2024-11-21T21:15:40,432 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2858 2024-11-21T21:15:40,432 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2901 2024-11-21T21:15:40,432 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2873 2024-11-21T21:15:40,432 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T21:15:40,432 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T21:15:40,432 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T21:15:40,432 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:40,432 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:40,432 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:15:40,434 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-21T21:15:40,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T21:15:40,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-21T21:15:40,440 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732223740439"}]},"ts":"1732223740439"} 2024-11-21T21:15:40,442 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T21:15:40,442 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T21:15:40,443 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T21:15:40,445 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4a01a47e5e966934386d5c50a408380a, UNASSIGN}] 2024-11-21T21:15:40,446 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4a01a47e5e966934386d5c50a408380a, UNASSIGN 2024-11-21T21:15:40,447 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=4a01a47e5e966934386d5c50a408380a, regionState=CLOSING, regionLocation=849dee24a242,41213,1732223653469 2024-11-21T21:15:40,448 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4a01a47e5e966934386d5c50a408380a, UNASSIGN because future has completed 2024-11-21T21:15:40,449 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T21:15:40,449 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469}] 2024-11-21T21:15:40,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-21T21:15:40,602 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(122): Close 4a01a47e5e966934386d5c50a408380a 2024-11-21T21:15:40,603 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T21:15:40,603 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1722): Closing 4a01a47e5e966934386d5c50a408380a, disabling compactions & flushes 2024-11-21T21:15:40,603 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:40,605 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=545 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/0d72e94cefaa43a9b2860828c5dce153 2024-11-21T21:15:40,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/275c6430018a4055bb4349174cf90acd as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/275c6430018a4055bb4349174cf90acd 2024-11-21T21:15:40,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/275c6430018a4055bb4349174cf90acd, entries=150, sequenceid=545, filesize=12.0 K 2024-11-21T21:15:40,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/771c09ff092e421b8b51c7bc415d7354 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/771c09ff092e421b8b51c7bc415d7354 2024-11-21T21:15:40,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/771c09ff092e421b8b51c7bc415d7354, entries=150, sequenceid=545, filesize=12.0 K 2024-11-21T21:15:40,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/0d72e94cefaa43a9b2860828c5dce153 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/0d72e94cefaa43a9b2860828c5dce153 2024-11-21T21:15:40,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/0d72e94cefaa43a9b2860828c5dce153, entries=150, sequenceid=545, filesize=12.0 K 2024-11-21T21:15:40,629 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=20.13 KB/20610 for 4a01a47e5e966934386d5c50a408380a in 857ms, sequenceid=545, compaction requested=false 2024-11-21T21:15:40,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a01a47e5e966934386d5c50a408380a: 2024-11-21T21:15:40,630 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:40,630 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:40,630 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. after waiting 0 ms 2024-11-21T21:15:40,630 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:40,630 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(2902): Flushing 4a01a47e5e966934386d5c50a408380a 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-21T21:15:40,630 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=A 2024-11-21T21:15:40,630 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:40,630 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=B 2024-11-21T21:15:40,630 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:40,630 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a01a47e5e966934386d5c50a408380a, store=C 2024-11-21T21:15:40,630 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:40,634 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/62ea9bd5e8004855a2c2f298d0baa102 is 50, key is test_row_0/A:col10/1732223739781/Put/seqid=0 2024-11-21T21:15:40,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742174_1350 (size=9857) 2024-11-21T21:15:40,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-21T21:15:41,040 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=554 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/62ea9bd5e8004855a2c2f298d0baa102 2024-11-21T21:15:41,053 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/d10afcb449fd426cb17189e5c60c869f is 50, key is test_row_0/B:col10/1732223739781/Put/seqid=0 2024-11-21T21:15:41,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742175_1351 (size=9857) 2024-11-21T21:15:41,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-21T21:15:41,458 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=554 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/d10afcb449fd426cb17189e5c60c869f 2024-11-21T21:15:41,510 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/605cbdca857249adbc808cb1359b27a6 is 50, key is test_row_0/C:col10/1732223739781/Put/seqid=0 2024-11-21T21:15:41,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742176_1352 (size=9857) 2024-11-21T21:15:41,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-21T21:15:41,603 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T21:15:41,915 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=554 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/605cbdca857249adbc808cb1359b27a6 2024-11-21T21:15:41,919 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/A/62ea9bd5e8004855a2c2f298d0baa102 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/62ea9bd5e8004855a2c2f298d0baa102 2024-11-21T21:15:41,925 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/62ea9bd5e8004855a2c2f298d0baa102, entries=100, sequenceid=554, filesize=9.6 K 2024-11-21T21:15:41,926 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/B/d10afcb449fd426cb17189e5c60c869f as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/d10afcb449fd426cb17189e5c60c869f 2024-11-21T21:15:41,931 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/d10afcb449fd426cb17189e5c60c869f, entries=100, sequenceid=554, filesize=9.6 K 2024-11-21T21:15:41,931 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/.tmp/C/605cbdca857249adbc808cb1359b27a6 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/605cbdca857249adbc808cb1359b27a6 2024-11-21T21:15:41,935 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/605cbdca857249adbc808cb1359b27a6, entries=100, sequenceid=554, filesize=9.6 K 2024-11-21T21:15:41,936 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 4a01a47e5e966934386d5c50a408380a in 1306ms, sequenceid=554, compaction requested=true 2024-11-21T21:15:41,936 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/a455f76a627548e7b993e7e3547668d2, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/9048cd595602481bb4fe7f0922900b3b, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/0db85a7d6bfe4e72a53f2e40e39f98bb, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/98a10f6d966045a889d61a74cac91223, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/31e15b07410b456686c42453c98583a9, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/a737dea74a4c48c38041c17276fa6eb0, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/5f284319c337491ca3e8f69ddf82201d, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/3e2169db2e9e469ba85af447192951f3, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/ebf39cb4c7da43c3b0af0ad5a2737c19, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/9a11bdb8b14e40c397f072bb9cccf302, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/3d0f1973382f4006829bca962612fd82, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/8cd894104d1a4dc0bb781a95a4fcc784, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/a4f35c4f18e8435d82c6dc4fd5068a7c, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/2c45a49a279c4fb4987594bb98f021dd, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/d661c5c8fb954138ba84e4f96f12a887, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/b78c7a36d44844399c9ed6bfd9543ab8, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/b727995f8a7f46f68d7834f534373a6e, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/91fd1e765e074c4eac77941394bdad3c, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/788f15e5ecbf4ad2980a5f0595957f26, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/f9437f9f78f64e139a2139e452548dd2, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/6adc7ef1edb44c26b93503a4b6db3717, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/84e10c37a1c345b68bfb079dee8b309f, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/66b42356cf3044d8bc8b0d46953480c4, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/f7fa999fbdd94c61a31b30c6fc40b17e, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/774676ec24d04102b570ae24617068d6, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/64f2c7edaef0402ea70f77c6fcad0736, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/717540fd17ef4d9fb820b2516c04e2b4, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/d2eaa8ec1d7a492aac941c69a9f84e9e, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/734eb414efc64976a33b0d885f061622, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/e4e8376788794bb1b567cbc7468b597f, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/7fb184da73f940bcbfc75c1f232a6137, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/a40f283cfdf248df8b6c9fe6cbb65adb, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/2f5c3b635cdd468bbd3b23cd6dc72b9b, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/39ab429e6e5e4545845702529f862b28, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/f4d0b3b2d9b4489fbf18d79ccf6c9950, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/11872f13a5134943b12a01bf05a7c015] to archive 2024-11-21T21:15:41,937 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T21:15:41,939 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/a455f76a627548e7b993e7e3547668d2 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/a455f76a627548e7b993e7e3547668d2 2024-11-21T21:15:41,941 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/9048cd595602481bb4fe7f0922900b3b to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/9048cd595602481bb4fe7f0922900b3b 2024-11-21T21:15:41,942 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/0db85a7d6bfe4e72a53f2e40e39f98bb to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/0db85a7d6bfe4e72a53f2e40e39f98bb 2024-11-21T21:15:41,943 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/98a10f6d966045a889d61a74cac91223 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/98a10f6d966045a889d61a74cac91223 2024-11-21T21:15:41,944 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/31e15b07410b456686c42453c98583a9 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/31e15b07410b456686c42453c98583a9 2024-11-21T21:15:41,945 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/a737dea74a4c48c38041c17276fa6eb0 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/a737dea74a4c48c38041c17276fa6eb0 2024-11-21T21:15:41,946 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/5f284319c337491ca3e8f69ddf82201d to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/5f284319c337491ca3e8f69ddf82201d 2024-11-21T21:15:41,947 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/3e2169db2e9e469ba85af447192951f3 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/3e2169db2e9e469ba85af447192951f3 2024-11-21T21:15:41,948 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/ebf39cb4c7da43c3b0af0ad5a2737c19 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/ebf39cb4c7da43c3b0af0ad5a2737c19 2024-11-21T21:15:41,949 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/9a11bdb8b14e40c397f072bb9cccf302 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/9a11bdb8b14e40c397f072bb9cccf302 2024-11-21T21:15:41,950 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/3d0f1973382f4006829bca962612fd82 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/3d0f1973382f4006829bca962612fd82 2024-11-21T21:15:41,950 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/8cd894104d1a4dc0bb781a95a4fcc784 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/8cd894104d1a4dc0bb781a95a4fcc784 2024-11-21T21:15:41,951 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/a4f35c4f18e8435d82c6dc4fd5068a7c to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/a4f35c4f18e8435d82c6dc4fd5068a7c 2024-11-21T21:15:41,952 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/2c45a49a279c4fb4987594bb98f021dd to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/2c45a49a279c4fb4987594bb98f021dd 2024-11-21T21:15:41,953 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/d661c5c8fb954138ba84e4f96f12a887 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/d661c5c8fb954138ba84e4f96f12a887 2024-11-21T21:15:41,954 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/b78c7a36d44844399c9ed6bfd9543ab8 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/b78c7a36d44844399c9ed6bfd9543ab8 2024-11-21T21:15:41,954 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/b727995f8a7f46f68d7834f534373a6e to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/b727995f8a7f46f68d7834f534373a6e 2024-11-21T21:15:41,955 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/91fd1e765e074c4eac77941394bdad3c to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/91fd1e765e074c4eac77941394bdad3c 2024-11-21T21:15:41,956 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/788f15e5ecbf4ad2980a5f0595957f26 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/788f15e5ecbf4ad2980a5f0595957f26 2024-11-21T21:15:41,957 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/f9437f9f78f64e139a2139e452548dd2 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/f9437f9f78f64e139a2139e452548dd2 2024-11-21T21:15:41,957 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/6adc7ef1edb44c26b93503a4b6db3717 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/6adc7ef1edb44c26b93503a4b6db3717 2024-11-21T21:15:41,958 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/84e10c37a1c345b68bfb079dee8b309f to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/84e10c37a1c345b68bfb079dee8b309f 2024-11-21T21:15:41,959 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/66b42356cf3044d8bc8b0d46953480c4 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/66b42356cf3044d8bc8b0d46953480c4 2024-11-21T21:15:41,959 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/f7fa999fbdd94c61a31b30c6fc40b17e to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/f7fa999fbdd94c61a31b30c6fc40b17e 2024-11-21T21:15:41,960 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/774676ec24d04102b570ae24617068d6 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/774676ec24d04102b570ae24617068d6 2024-11-21T21:15:41,961 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/64f2c7edaef0402ea70f77c6fcad0736 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/64f2c7edaef0402ea70f77c6fcad0736 2024-11-21T21:15:41,962 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/717540fd17ef4d9fb820b2516c04e2b4 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/717540fd17ef4d9fb820b2516c04e2b4 2024-11-21T21:15:41,963 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/d2eaa8ec1d7a492aac941c69a9f84e9e to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/d2eaa8ec1d7a492aac941c69a9f84e9e 2024-11-21T21:15:41,963 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/734eb414efc64976a33b0d885f061622 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/734eb414efc64976a33b0d885f061622 2024-11-21T21:15:41,964 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/e4e8376788794bb1b567cbc7468b597f to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/e4e8376788794bb1b567cbc7468b597f 2024-11-21T21:15:41,965 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/7fb184da73f940bcbfc75c1f232a6137 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/7fb184da73f940bcbfc75c1f232a6137 2024-11-21T21:15:41,966 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/a40f283cfdf248df8b6c9fe6cbb65adb to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/a40f283cfdf248df8b6c9fe6cbb65adb 2024-11-21T21:15:41,966 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/2f5c3b635cdd468bbd3b23cd6dc72b9b to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/2f5c3b635cdd468bbd3b23cd6dc72b9b 2024-11-21T21:15:41,967 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/39ab429e6e5e4545845702529f862b28 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/39ab429e6e5e4545845702529f862b28 2024-11-21T21:15:41,968 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/f4d0b3b2d9b4489fbf18d79ccf6c9950 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/f4d0b3b2d9b4489fbf18d79ccf6c9950 2024-11-21T21:15:41,969 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/11872f13a5134943b12a01bf05a7c015 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/11872f13a5134943b12a01bf05a7c015 2024-11-21T21:15:41,970 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/9eecf6bf0d0b424ab50005316b35b16c, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/b445f33fb2b3411e90ad7684d7c7f2f2, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/e529b5857bb14522b308609d2e5a7831, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/1c53bd25143b40bba76313bf6681a599, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/a5806a032c1e4ef6ac931fcbdfd8fac7, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/62f69a709d26449ab44f4fe32057d1f8, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/cc48533822ba4b719815a7b96fb96f11, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/a2506d22634b43be9f4f11f3aa29b30d, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/d4537c9d37734294817c9f66cfff6df4, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/5568f8c6f67e47058878abc7e00eb217, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/fe5cda48c84f4303ba75bd4426961040, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/f17b05b8897a45b38d88f45bff84d08d, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/ae0146d508bc4f6fa0c54ca62a031d0f, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/ae0e9f578ee54753b8490818a0e969e4, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/6b28d618aa2b4c349314ab6a45b913fd, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/dfe3e676bbb64e158ec9866b8696efe6, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/6c4cb720c43b424f8de2e550f9141f53, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/b7bacde0e1fd4fbf8743fbc8c7d2f476, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/7cf1924b1a8e40779c271f3e4599b2a4, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/3dc203bbdf984efaac203f7205869a6f, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/bc58f85f41334cbfb7a845ab8966cd82, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/1fbc739d6f0f4128bb2eac5db6933e4f, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/8257097dd9044a65ae21265da2228663, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/7281d436b7b04f6993639981e0eb88b3, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/40e9ec4e721c4710b6a6d8b1853bfdb6, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/ff1e6dda46f84b15837f8f46c634acd9, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/25bdf7f75fb1430598e7e2c4ed456e64, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/1ac37caa6e9445fc95f3040dec2b9cd0, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/4c7a10b72d6c4cbda0647934d8491963, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/59c07620829e481c9f1d7ac8d0aa9973, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/bf823efb7fcb4ef88cfdac6715eec433, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/1513d410f27d4b5e98a3d8824eee261d, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/8fc06e51021640309a52925ad1a4e56a, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/f81bb18b1f40493aa7a8f23ec94582f8, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/d96ba76cbcc541b8b612ae86e85bc18f, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/5203815b13444d2580a961544401af9f] to archive 2024-11-21T21:15:41,971 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T21:15:41,972 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/9eecf6bf0d0b424ab50005316b35b16c to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/9eecf6bf0d0b424ab50005316b35b16c 2024-11-21T21:15:41,973 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/b445f33fb2b3411e90ad7684d7c7f2f2 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/b445f33fb2b3411e90ad7684d7c7f2f2 2024-11-21T21:15:41,974 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/e529b5857bb14522b308609d2e5a7831 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/e529b5857bb14522b308609d2e5a7831 2024-11-21T21:15:41,975 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/1c53bd25143b40bba76313bf6681a599 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/1c53bd25143b40bba76313bf6681a599 2024-11-21T21:15:41,976 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/a5806a032c1e4ef6ac931fcbdfd8fac7 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/a5806a032c1e4ef6ac931fcbdfd8fac7 2024-11-21T21:15:41,977 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/62f69a709d26449ab44f4fe32057d1f8 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/62f69a709d26449ab44f4fe32057d1f8 2024-11-21T21:15:41,977 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/cc48533822ba4b719815a7b96fb96f11 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/cc48533822ba4b719815a7b96fb96f11 2024-11-21T21:15:41,978 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/a2506d22634b43be9f4f11f3aa29b30d to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/a2506d22634b43be9f4f11f3aa29b30d 2024-11-21T21:15:41,979 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/d4537c9d37734294817c9f66cfff6df4 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/d4537c9d37734294817c9f66cfff6df4 2024-11-21T21:15:41,980 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/5568f8c6f67e47058878abc7e00eb217 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/5568f8c6f67e47058878abc7e00eb217 2024-11-21T21:15:41,981 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/fe5cda48c84f4303ba75bd4426961040 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/fe5cda48c84f4303ba75bd4426961040 2024-11-21T21:15:41,982 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/f17b05b8897a45b38d88f45bff84d08d to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/f17b05b8897a45b38d88f45bff84d08d 2024-11-21T21:15:41,982 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/ae0146d508bc4f6fa0c54ca62a031d0f to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/ae0146d508bc4f6fa0c54ca62a031d0f 2024-11-21T21:15:41,983 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/ae0e9f578ee54753b8490818a0e969e4 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/ae0e9f578ee54753b8490818a0e969e4 2024-11-21T21:15:41,984 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/6b28d618aa2b4c349314ab6a45b913fd to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/6b28d618aa2b4c349314ab6a45b913fd 2024-11-21T21:15:41,985 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/dfe3e676bbb64e158ec9866b8696efe6 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/dfe3e676bbb64e158ec9866b8696efe6 2024-11-21T21:15:41,986 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/6c4cb720c43b424f8de2e550f9141f53 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/6c4cb720c43b424f8de2e550f9141f53 2024-11-21T21:15:41,986 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/b7bacde0e1fd4fbf8743fbc8c7d2f476 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/b7bacde0e1fd4fbf8743fbc8c7d2f476 2024-11-21T21:15:41,987 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/7cf1924b1a8e40779c271f3e4599b2a4 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/7cf1924b1a8e40779c271f3e4599b2a4 2024-11-21T21:15:41,988 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/3dc203bbdf984efaac203f7205869a6f to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/3dc203bbdf984efaac203f7205869a6f 2024-11-21T21:15:41,989 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/bc58f85f41334cbfb7a845ab8966cd82 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/bc58f85f41334cbfb7a845ab8966cd82 2024-11-21T21:15:41,990 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/1fbc739d6f0f4128bb2eac5db6933e4f to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/1fbc739d6f0f4128bb2eac5db6933e4f 2024-11-21T21:15:41,991 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/8257097dd9044a65ae21265da2228663 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/8257097dd9044a65ae21265da2228663 2024-11-21T21:15:41,992 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/7281d436b7b04f6993639981e0eb88b3 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/7281d436b7b04f6993639981e0eb88b3 2024-11-21T21:15:41,992 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/40e9ec4e721c4710b6a6d8b1853bfdb6 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/40e9ec4e721c4710b6a6d8b1853bfdb6 2024-11-21T21:15:41,993 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/ff1e6dda46f84b15837f8f46c634acd9 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/ff1e6dda46f84b15837f8f46c634acd9 2024-11-21T21:15:41,994 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/25bdf7f75fb1430598e7e2c4ed456e64 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/25bdf7f75fb1430598e7e2c4ed456e64 2024-11-21T21:15:41,995 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/1ac37caa6e9445fc95f3040dec2b9cd0 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/1ac37caa6e9445fc95f3040dec2b9cd0 2024-11-21T21:15:41,996 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/4c7a10b72d6c4cbda0647934d8491963 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/4c7a10b72d6c4cbda0647934d8491963 2024-11-21T21:15:41,997 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/59c07620829e481c9f1d7ac8d0aa9973 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/59c07620829e481c9f1d7ac8d0aa9973 2024-11-21T21:15:41,997 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/bf823efb7fcb4ef88cfdac6715eec433 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/bf823efb7fcb4ef88cfdac6715eec433 2024-11-21T21:15:41,998 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/1513d410f27d4b5e98a3d8824eee261d to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/1513d410f27d4b5e98a3d8824eee261d 2024-11-21T21:15:41,999 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/8fc06e51021640309a52925ad1a4e56a to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/8fc06e51021640309a52925ad1a4e56a 2024-11-21T21:15:42,000 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/f81bb18b1f40493aa7a8f23ec94582f8 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/f81bb18b1f40493aa7a8f23ec94582f8 2024-11-21T21:15:42,000 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/d96ba76cbcc541b8b612ae86e85bc18f to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/d96ba76cbcc541b8b612ae86e85bc18f 2024-11-21T21:15:42,001 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/5203815b13444d2580a961544401af9f to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/5203815b13444d2580a961544401af9f 2024-11-21T21:15:42,003 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/e512e73b5f35445eaddfcfb9e3ad1691, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/adbdbe7c41494e6f87a29c5dfa0a67a8, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/c0e15157cd344c6188bf5ad33d8d06f0, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/f2409a7e5fd5477ab565ec6aef7ee4d1, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/34214f256ddf49b29f7a8ac453f0155a, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/9054e285b9764831bb977ce50f8fbadc, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/835014565f2647178301bd67ab9dbe0a, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/dc2fbbbb5dec40ed83bfbb6d96477b20, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/ab317961abc74c87a714dd75e56f47ce, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/d8a67e21e6164c8387426b2c4c11314b, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/cc6b85b8725e4ce2af48e49d51a17907, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/bce5506fac424df68ff76a881d6eebb9, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/7e18558d45e947f2bbf657676bd833c5, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/5dbaeb3eff5348abb6b8e89370a2208e, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/1185a922c7ee43b4bf5795ce00cd83c3, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/7a9d28bc952b4d10ad6095a7593b9302, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/c2de8df81c0344fcbe3335dc2710d193, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/63b0514f2a084dfbaad1801e1c735726, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/2101faf1cf31476f8202f261b5d19cdb, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/66552d099e3c4f44aeb832395a13f4c8, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/d02926a172834ebaa159bd9aa5ad27af, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/f044800eb8534605a24ea606968811be, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/7170f3aef45444819eff3a344734acc9, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/89c3dba9870f4b7689e35474afa5e79b, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/6d18fcb42076447085636c979ac93a9a, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/f95ffed44a6047919d1fd1cda00aabc9, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/dabaf0c6ca1f4b99ae31125f35782b92, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/89eb9f60e9e2423da1cfc49c174f43b6, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/ac4f37ccdbc344ffbd4a156a7372f664, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/d0c5290ee4554826b9f2dc2efc6cdfc0, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/5fabd25464fd495a9e02b16f51ab78d3, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/fda8dab5b805458fb9d0e61fe1aa33f3, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/44b674a43daf4f33b69d70b5b07fa533, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/e97f9d7dfab5490a9a540a27af7e03ba, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/4e6b1d5755694888a719f42a3e230a80] to archive 2024-11-21T21:15:42,004 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T21:15:42,005 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/e512e73b5f35445eaddfcfb9e3ad1691 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/e512e73b5f35445eaddfcfb9e3ad1691 2024-11-21T21:15:42,006 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/adbdbe7c41494e6f87a29c5dfa0a67a8 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/adbdbe7c41494e6f87a29c5dfa0a67a8 2024-11-21T21:15:42,007 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/c0e15157cd344c6188bf5ad33d8d06f0 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/c0e15157cd344c6188bf5ad33d8d06f0 2024-11-21T21:15:42,008 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/f2409a7e5fd5477ab565ec6aef7ee4d1 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/f2409a7e5fd5477ab565ec6aef7ee4d1 2024-11-21T21:15:42,010 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/34214f256ddf49b29f7a8ac453f0155a to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/34214f256ddf49b29f7a8ac453f0155a 2024-11-21T21:15:42,011 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/9054e285b9764831bb977ce50f8fbadc to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/9054e285b9764831bb977ce50f8fbadc 2024-11-21T21:15:42,012 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/835014565f2647178301bd67ab9dbe0a to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/835014565f2647178301bd67ab9dbe0a 2024-11-21T21:15:42,013 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/dc2fbbbb5dec40ed83bfbb6d96477b20 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/dc2fbbbb5dec40ed83bfbb6d96477b20 2024-11-21T21:15:42,014 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/ab317961abc74c87a714dd75e56f47ce to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/ab317961abc74c87a714dd75e56f47ce 2024-11-21T21:15:42,015 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/d8a67e21e6164c8387426b2c4c11314b to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/d8a67e21e6164c8387426b2c4c11314b 2024-11-21T21:15:42,016 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/cc6b85b8725e4ce2af48e49d51a17907 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/cc6b85b8725e4ce2af48e49d51a17907 2024-11-21T21:15:42,017 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/bce5506fac424df68ff76a881d6eebb9 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/bce5506fac424df68ff76a881d6eebb9 2024-11-21T21:15:42,018 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/7e18558d45e947f2bbf657676bd833c5 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/7e18558d45e947f2bbf657676bd833c5 2024-11-21T21:15:42,019 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/5dbaeb3eff5348abb6b8e89370a2208e to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/5dbaeb3eff5348abb6b8e89370a2208e 2024-11-21T21:15:42,019 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/1185a922c7ee43b4bf5795ce00cd83c3 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/1185a922c7ee43b4bf5795ce00cd83c3 2024-11-21T21:15:42,020 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/7a9d28bc952b4d10ad6095a7593b9302 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/7a9d28bc952b4d10ad6095a7593b9302 2024-11-21T21:15:42,021 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/c2de8df81c0344fcbe3335dc2710d193 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/c2de8df81c0344fcbe3335dc2710d193 2024-11-21T21:15:42,022 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/63b0514f2a084dfbaad1801e1c735726 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/63b0514f2a084dfbaad1801e1c735726 2024-11-21T21:15:42,022 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/2101faf1cf31476f8202f261b5d19cdb to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/2101faf1cf31476f8202f261b5d19cdb 2024-11-21T21:15:42,023 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/66552d099e3c4f44aeb832395a13f4c8 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/66552d099e3c4f44aeb832395a13f4c8 2024-11-21T21:15:42,024 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/d02926a172834ebaa159bd9aa5ad27af to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/d02926a172834ebaa159bd9aa5ad27af 2024-11-21T21:15:42,025 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/f044800eb8534605a24ea606968811be to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/f044800eb8534605a24ea606968811be 2024-11-21T21:15:42,026 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/7170f3aef45444819eff3a344734acc9 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/7170f3aef45444819eff3a344734acc9 2024-11-21T21:15:42,026 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/89c3dba9870f4b7689e35474afa5e79b to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/89c3dba9870f4b7689e35474afa5e79b 2024-11-21T21:15:42,027 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/6d18fcb42076447085636c979ac93a9a to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/6d18fcb42076447085636c979ac93a9a 2024-11-21T21:15:42,028 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/f95ffed44a6047919d1fd1cda00aabc9 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/f95ffed44a6047919d1fd1cda00aabc9 2024-11-21T21:15:42,029 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/dabaf0c6ca1f4b99ae31125f35782b92 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/dabaf0c6ca1f4b99ae31125f35782b92 2024-11-21T21:15:42,030 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/89eb9f60e9e2423da1cfc49c174f43b6 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/89eb9f60e9e2423da1cfc49c174f43b6 2024-11-21T21:15:42,030 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/ac4f37ccdbc344ffbd4a156a7372f664 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/ac4f37ccdbc344ffbd4a156a7372f664 2024-11-21T21:15:42,031 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/d0c5290ee4554826b9f2dc2efc6cdfc0 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/d0c5290ee4554826b9f2dc2efc6cdfc0 2024-11-21T21:15:42,032 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/5fabd25464fd495a9e02b16f51ab78d3 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/5fabd25464fd495a9e02b16f51ab78d3 2024-11-21T21:15:42,033 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/fda8dab5b805458fb9d0e61fe1aa33f3 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/fda8dab5b805458fb9d0e61fe1aa33f3 2024-11-21T21:15:42,034 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/44b674a43daf4f33b69d70b5b07fa533 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/44b674a43daf4f33b69d70b5b07fa533 2024-11-21T21:15:42,035 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/e97f9d7dfab5490a9a540a27af7e03ba to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/e97f9d7dfab5490a9a540a27af7e03ba 2024-11-21T21:15:42,035 DEBUG [StoreCloser-TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/4e6b1d5755694888a719f42a3e230a80 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/4e6b1d5755694888a719f42a3e230a80 2024-11-21T21:15:42,039 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/recovered.edits/557.seqid, newMaxSeqId=557, maxSeqId=1 2024-11-21T21:15:42,040 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. 2024-11-21T21:15:42,040 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1676): Region close journal for 4a01a47e5e966934386d5c50a408380a: Waiting for close lock at 1732223740603Running coprocessor pre-close hooks at 1732223740603Disabling compacts and flushes for region at 1732223740603Disabling writes for close at 1732223740630 (+27 ms)Obtaining lock to block concurrent updates at 1732223740630Preparing flush snapshotting stores in 4a01a47e5e966934386d5c50a408380a at 1732223740630Finished memstore snapshotting TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1732223740630Flushing stores of TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a. at 1732223740631 (+1 ms)Flushing 4a01a47e5e966934386d5c50a408380a/A: creating writer at 1732223740631Flushing 4a01a47e5e966934386d5c50a408380a/A: appending metadata at 1732223740634 (+3 ms)Flushing 4a01a47e5e966934386d5c50a408380a/A: closing flushed file at 1732223740634Flushing 4a01a47e5e966934386d5c50a408380a/B: creating writer at 1732223741049 (+415 ms)Flushing 4a01a47e5e966934386d5c50a408380a/B: appending metadata at 1732223741053 (+4 ms)Flushing 4a01a47e5e966934386d5c50a408380a/B: closing flushed file at 1732223741053Flushing 4a01a47e5e966934386d5c50a408380a/C: creating writer at 1732223741507 (+454 ms)Flushing 4a01a47e5e966934386d5c50a408380a/C: appending metadata at 1732223741509 (+2 ms)Flushing 4a01a47e5e966934386d5c50a408380a/C: closing flushed file at 1732223741510 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39af4894: reopening flushed file at 1732223741918 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7935f444: reopening flushed file at 1732223741925 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c631b5: reopening flushed file at 1732223741931 (+6 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 4a01a47e5e966934386d5c50a408380a in 1306ms, sequenceid=554, compaction requested=true at 1732223741936 (+5 ms)Writing region close event to WAL at 1732223742037 (+101 ms)Running coprocessor post-close hooks at 1732223742040 (+3 ms)Closed at 1732223742040 2024-11-21T21:15:42,042 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(157): Closed 4a01a47e5e966934386d5c50a408380a 2024-11-21T21:15:42,042 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=4a01a47e5e966934386d5c50a408380a, regionState=CLOSED 2024-11-21T21:15:42,043 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 because future has completed 2024-11-21T21:15:42,046 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-21T21:15:42,046 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseRegionProcedure 4a01a47e5e966934386d5c50a408380a, server=849dee24a242,41213,1732223653469 in 1.5950 sec 2024-11-21T21:15:42,047 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-21T21:15:42,047 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4a01a47e5e966934386d5c50a408380a, UNASSIGN in 1.6010 sec 2024-11-21T21:15:42,049 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-21T21:15:42,049 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6060 sec 2024-11-21T21:15:42,050 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732223742050"}]},"ts":"1732223742050"} 2024-11-21T21:15:42,051 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T21:15:42,051 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T21:15:42,053 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6180 sec 2024-11-21T21:15:42,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-21T21:15:42,582 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:15:42,584 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-21T21:15:42,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:15:42,586 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=92, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:15:42,587 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=92, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:15:42,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-21T21:15:42,590 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a 2024-11-21T21:15:42,593 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A, FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B, FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C, FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/recovered.edits] 2024-11-21T21:15:42,596 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/275c6430018a4055bb4349174cf90acd to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/275c6430018a4055bb4349174cf90acd 2024-11-21T21:15:42,597 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/62ea9bd5e8004855a2c2f298d0baa102 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/62ea9bd5e8004855a2c2f298d0baa102 2024-11-21T21:15:42,598 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/e25f7a26e3c54d2c99a5bab456b688c9 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/A/e25f7a26e3c54d2c99a5bab456b688c9 2024-11-21T21:15:42,600 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/771c09ff092e421b8b51c7bc415d7354 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/771c09ff092e421b8b51c7bc415d7354 2024-11-21T21:15:42,601 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/a284e07630594afc9b07d2d6d83235d9 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/a284e07630594afc9b07d2d6d83235d9 2024-11-21T21:15:42,602 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/d10afcb449fd426cb17189e5c60c869f to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/B/d10afcb449fd426cb17189e5c60c869f 2024-11-21T21:15:42,603 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/0d72e94cefaa43a9b2860828c5dce153 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/0d72e94cefaa43a9b2860828c5dce153 2024-11-21T21:15:42,604 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/605cbdca857249adbc808cb1359b27a6 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/605cbdca857249adbc808cb1359b27a6 2024-11-21T21:15:42,604 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/b2ca71a40904495ab7bcb7f59ad3ba15 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/C/b2ca71a40904495ab7bcb7f59ad3ba15 2024-11-21T21:15:42,606 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/recovered.edits/557.seqid to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a/recovered.edits/557.seqid 2024-11-21T21:15:42,606 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/4a01a47e5e966934386d5c50a408380a 2024-11-21T21:15:42,606 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T21:15:42,608 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=92, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:15:42,611 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T21:15:42,612 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T21:15:42,614 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=92, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:15:42,614 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T21:15:42,614 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732223742614"}]},"ts":"9223372036854775807"} 2024-11-21T21:15:42,616 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T21:15:42,616 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 4a01a47e5e966934386d5c50a408380a, NAME => 'TestAcidGuarantees,,1732223716778.4a01a47e5e966934386d5c50a408380a.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T21:15:42,616 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T21:15:42,616 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732223742616"}]},"ts":"9223372036854775807"} 2024-11-21T21:15:42,617 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T21:15:42,618 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=92, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:15:42,619 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 34 msec 2024-11-21T21:15:42,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-21T21:15:42,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T21:15:42,691 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:15:42,704 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=250 (was 248) - Thread LEAK? -, OpenFileDescriptor=453 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=392 (was 341) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=12050 (was 11484) - AvailableMemoryMB LEAK? - 2024-11-21T21:15:42,713 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=392, ProcessCount=11, AvailableMemoryMB=12050 2024-11-21T21:15:42,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T21:15:42,715 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T21:15:42,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T21:15:42,716 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T21:15:42,717 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:15:42,717 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 93 2024-11-21T21:15:42,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-21T21:15:42,717 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T21:15:42,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742177_1353 (size=1061) 2024-11-21T21:15:42,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-21T21:15:43,010 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-21T21:15:43,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-21T21:15:43,129 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f 2024-11-21T21:15:43,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742178_1354 (size=53) 2024-11-21T21:15:43,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-21T21:15:43,541 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:15:43,541 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ca977aeb13804530742b16bc2c00bf08, disabling compactions & flushes 2024-11-21T21:15:43,541 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:43,541 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:43,541 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. after waiting 0 ms 2024-11-21T21:15:43,541 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:43,541 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:43,541 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ca977aeb13804530742b16bc2c00bf08: Waiting for close lock at 1732223743541Disabling compacts and flushes for region at 1732223743541Disabling writes for close at 1732223743541Writing region close event to WAL at 1732223743541Closed at 1732223743541 2024-11-21T21:15:43,542 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T21:15:43,542 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732223743542"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732223743542"}]},"ts":"1732223743542"} 2024-11-21T21:15:43,543 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T21:15:43,544 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T21:15:43,544 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732223743544"}]},"ts":"1732223743544"} 2024-11-21T21:15:43,545 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T21:15:43,546 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca977aeb13804530742b16bc2c00bf08, ASSIGN}] 2024-11-21T21:15:43,547 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca977aeb13804530742b16bc2c00bf08, ASSIGN 2024-11-21T21:15:43,547 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca977aeb13804530742b16bc2c00bf08, ASSIGN; state=OFFLINE, location=849dee24a242,41213,1732223653469; forceNewPlan=false, retain=false 2024-11-21T21:15:43,698 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=ca977aeb13804530742b16bc2c00bf08, regionState=OPENING, regionLocation=849dee24a242,41213,1732223653469 2024-11-21T21:15:43,701 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca977aeb13804530742b16bc2c00bf08, ASSIGN because future has completed 2024-11-21T21:15:43,701 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469}] 2024-11-21T21:15:43,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-21T21:15:43,857 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:43,857 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7752): Opening region: {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} 2024-11-21T21:15:43,858 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:43,858 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:15:43,858 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7794): checking encryption for ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:43,858 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7797): checking classloading for ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:43,860 INFO [StoreOpener-ca977aeb13804530742b16bc2c00bf08-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:43,861 INFO [StoreOpener-ca977aeb13804530742b16bc2c00bf08-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T21:15:43,861 INFO [StoreOpener-ca977aeb13804530742b16bc2c00bf08-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca977aeb13804530742b16bc2c00bf08 columnFamilyName A 2024-11-21T21:15:43,861 DEBUG [StoreOpener-ca977aeb13804530742b16bc2c00bf08-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:15:43,862 INFO [StoreOpener-ca977aeb13804530742b16bc2c00bf08-1 {}] regionserver.HStore(327): Store=ca977aeb13804530742b16bc2c00bf08/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:15:43,862 INFO [StoreOpener-ca977aeb13804530742b16bc2c00bf08-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:43,863 INFO [StoreOpener-ca977aeb13804530742b16bc2c00bf08-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T21:15:43,863 INFO [StoreOpener-ca977aeb13804530742b16bc2c00bf08-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca977aeb13804530742b16bc2c00bf08 columnFamilyName B 2024-11-21T21:15:43,863 DEBUG [StoreOpener-ca977aeb13804530742b16bc2c00bf08-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:15:43,864 INFO [StoreOpener-ca977aeb13804530742b16bc2c00bf08-1 {}] regionserver.HStore(327): Store=ca977aeb13804530742b16bc2c00bf08/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:15:43,864 INFO [StoreOpener-ca977aeb13804530742b16bc2c00bf08-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:43,865 INFO [StoreOpener-ca977aeb13804530742b16bc2c00bf08-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T21:15:43,865 INFO [StoreOpener-ca977aeb13804530742b16bc2c00bf08-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca977aeb13804530742b16bc2c00bf08 columnFamilyName C 2024-11-21T21:15:43,865 DEBUG [StoreOpener-ca977aeb13804530742b16bc2c00bf08-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:15:43,866 INFO [StoreOpener-ca977aeb13804530742b16bc2c00bf08-1 {}] regionserver.HStore(327): Store=ca977aeb13804530742b16bc2c00bf08/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:15:43,866 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:43,866 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1038): replaying wal for ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:43,867 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:43,867 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:43,870 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1048): stopping wal replay for ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:43,870 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1060): Cleaning up temporary data for ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:43,870 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T21:15:43,872 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1093): writing seq id for ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:43,874 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T21:15:43,875 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1114): Opened ca977aeb13804530742b16bc2c00bf08; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70351220, jitterRate=0.04831486940383911}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T21:15:43,875 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:43,876 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1006): Region open journal for ca977aeb13804530742b16bc2c00bf08: Running coprocessor pre-open hook at 1732223743858Writing region info on filesystem at 1732223743858Initializing all the Stores at 1732223743859 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732223743859Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732223743859Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732223743859Cleaning up temporary data from old regions at 1732223743870 (+11 ms)Running coprocessor post-open hooks at 1732223743875 (+5 ms)Region opened successfully at 1732223743876 (+1 ms) 2024-11-21T21:15:43,876 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., pid=95, masterSystemTime=1732223743854 2024-11-21T21:15:43,878 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:43,878 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:43,878 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=ca977aeb13804530742b16bc2c00bf08, regionState=OPEN, openSeqNum=2, regionLocation=849dee24a242,41213,1732223653469 2024-11-21T21:15:43,879 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 because future has completed 2024-11-21T21:15:43,881 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-21T21:15:43,881 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; OpenRegionProcedure ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 in 179 msec 2024-11-21T21:15:43,882 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-21T21:15:43,882 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca977aeb13804530742b16bc2c00bf08, ASSIGN in 335 msec 2024-11-21T21:15:43,883 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T21:15:43,883 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732223743883"}]},"ts":"1732223743883"} 2024-11-21T21:15:43,884 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T21:15:43,884 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T21:15:43,885 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1690 sec 2024-11-21T21:15:44,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-21T21:15:44,860 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:15:44,861 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7db9998b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:44,862 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:15:44,862 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:15:44,863 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:15:44,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:15:44,864 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:15:44,864 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d9bb0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:44,864 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:15:44,864 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:15:44,864 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:44,866 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:15:44,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@535a0265, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:44,867 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:15:44,868 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:15:44,868 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:15:44,869 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44250, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:15:44,870 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T21:15:44,871 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 849dee24a242,42485,1732223652547 2024-11-21T21:15:44,871 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7e851c2f 2024-11-21T21:15:44,871 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T21:15:44,872 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49624, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T21:15:44,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T21:15:44,875 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T21:15:44,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=96, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-21T21:15:44,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742179_1355 (size=1097) 2024-11-21T21:15:44,906 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-21T21:15:44,906 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-21T21:15:44,910 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T21:15:44,915 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca977aeb13804530742b16bc2c00bf08, REOPEN/MOVE}] 2024-11-21T21:15:44,916 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca977aeb13804530742b16bc2c00bf08, REOPEN/MOVE 2024-11-21T21:15:44,918 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=ca977aeb13804530742b16bc2c00bf08, regionState=CLOSING, regionLocation=849dee24a242,41213,1732223653469 2024-11-21T21:15:44,920 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca977aeb13804530742b16bc2c00bf08, REOPEN/MOVE because future has completed 2024-11-21T21:15:44,920 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T21:15:44,920 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=99, ppid=98, state=RUNNABLE, hasLock=false; CloseRegionProcedure ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469}] 2024-11-21T21:15:45,073 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(122): Close ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:45,073 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T21:15:45,073 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1722): Closing ca977aeb13804530742b16bc2c00bf08, disabling compactions & flushes 2024-11-21T21:15:45,073 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:45,073 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:45,073 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. after waiting 0 ms 2024-11-21T21:15:45,073 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:45,076 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T21:15:45,077 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:45,077 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1676): Region close journal for ca977aeb13804530742b16bc2c00bf08: Waiting for close lock at 1732223745073Running coprocessor pre-close hooks at 1732223745073Disabling compacts and flushes for region at 1732223745073Disabling writes for close at 1732223745073Writing region close event to WAL at 1732223745073Running coprocessor post-close hooks at 1732223745077 (+4 ms)Closed at 1732223745077 2024-11-21T21:15:45,077 WARN [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegionServer(3299): Not adding moved region record: ca977aeb13804530742b16bc2c00bf08 to self. 2024-11-21T21:15:45,079 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(157): Closed ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:45,080 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=ca977aeb13804530742b16bc2c00bf08, regionState=CLOSED 2024-11-21T21:15:45,081 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=99, ppid=98, state=RUNNABLE, hasLock=false; CloseRegionProcedure ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 because future has completed 2024-11-21T21:15:45,084 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=99, resume processing ppid=98 2024-11-21T21:15:45,084 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, ppid=98, state=SUCCESS, hasLock=false; CloseRegionProcedure ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 in 162 msec 2024-11-21T21:15:45,085 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca977aeb13804530742b16bc2c00bf08, REOPEN/MOVE; state=CLOSED, location=849dee24a242,41213,1732223653469; forceNewPlan=false, retain=true 2024-11-21T21:15:45,236 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=ca977aeb13804530742b16bc2c00bf08, regionState=OPENING, regionLocation=849dee24a242,41213,1732223653469 2024-11-21T21:15:45,238 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca977aeb13804530742b16bc2c00bf08, REOPEN/MOVE because future has completed 2024-11-21T21:15:45,238 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469}] 2024-11-21T21:15:45,392 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:45,392 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7752): Opening region: {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} 2024-11-21T21:15:45,393 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:45,393 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:15:45,393 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7794): checking encryption for ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:45,393 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7797): checking classloading for ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:45,394 INFO [StoreOpener-ca977aeb13804530742b16bc2c00bf08-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:45,395 INFO [StoreOpener-ca977aeb13804530742b16bc2c00bf08-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T21:15:45,395 INFO [StoreOpener-ca977aeb13804530742b16bc2c00bf08-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca977aeb13804530742b16bc2c00bf08 columnFamilyName A 2024-11-21T21:15:45,396 DEBUG [StoreOpener-ca977aeb13804530742b16bc2c00bf08-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:15:45,396 INFO [StoreOpener-ca977aeb13804530742b16bc2c00bf08-1 {}] regionserver.HStore(327): Store=ca977aeb13804530742b16bc2c00bf08/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:15:45,396 INFO [StoreOpener-ca977aeb13804530742b16bc2c00bf08-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:45,397 INFO [StoreOpener-ca977aeb13804530742b16bc2c00bf08-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T21:15:45,397 INFO [StoreOpener-ca977aeb13804530742b16bc2c00bf08-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca977aeb13804530742b16bc2c00bf08 columnFamilyName B 2024-11-21T21:15:45,397 DEBUG [StoreOpener-ca977aeb13804530742b16bc2c00bf08-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:15:45,397 INFO [StoreOpener-ca977aeb13804530742b16bc2c00bf08-1 {}] regionserver.HStore(327): Store=ca977aeb13804530742b16bc2c00bf08/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:15:45,397 INFO [StoreOpener-ca977aeb13804530742b16bc2c00bf08-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:45,398 INFO [StoreOpener-ca977aeb13804530742b16bc2c00bf08-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T21:15:45,398 INFO [StoreOpener-ca977aeb13804530742b16bc2c00bf08-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca977aeb13804530742b16bc2c00bf08 columnFamilyName C 2024-11-21T21:15:45,398 DEBUG [StoreOpener-ca977aeb13804530742b16bc2c00bf08-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:15:45,398 INFO [StoreOpener-ca977aeb13804530742b16bc2c00bf08-1 {}] regionserver.HStore(327): Store=ca977aeb13804530742b16bc2c00bf08/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:15:45,398 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:45,398 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1038): replaying wal for ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:45,399 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:45,400 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:45,401 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1048): stopping wal replay for ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:45,401 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1060): Cleaning up temporary data for ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:45,401 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T21:15:45,403 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1093): writing seq id for ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:45,403 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1114): Opened ca977aeb13804530742b16bc2c00bf08; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66991572, jitterRate=-0.0017477869987487793}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T21:15:45,403 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:45,404 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1006): Region open journal for ca977aeb13804530742b16bc2c00bf08: Running coprocessor pre-open hook at 1732223745393Writing region info on filesystem at 1732223745393Initializing all the Stores at 1732223745394 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732223745394Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732223745394Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732223745394Cleaning up temporary data from old regions at 1732223745401 (+7 ms)Running coprocessor post-open hooks at 1732223745403 (+2 ms)Region opened successfully at 1732223745404 (+1 ms) 2024-11-21T21:15:45,405 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., pid=100, masterSystemTime=1732223745390 2024-11-21T21:15:45,407 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:45,407 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:45,407 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=ca977aeb13804530742b16bc2c00bf08, regionState=OPEN, openSeqNum=5, regionLocation=849dee24a242,41213,1732223653469 2024-11-21T21:15:45,408 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=100, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 because future has completed 2024-11-21T21:15:45,411 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=98 2024-11-21T21:15:45,411 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=98, state=SUCCESS, hasLock=false; OpenRegionProcedure ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 in 171 msec 2024-11-21T21:15:45,412 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-21T21:15:45,412 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca977aeb13804530742b16bc2c00bf08, REOPEN/MOVE in 496 msec 2024-11-21T21:15:45,414 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-11-21T21:15:45,414 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 502 msec 2024-11-21T21:15:45,416 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 540 msec 2024-11-21T21:15:45,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-21T21:15:45,417 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:15:45,417 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f0f1e03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:45,417 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:15:45,417 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:15:45,418 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:15:45,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:15:45,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:15:45,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b71db46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:45,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:15:45,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:15:45,419 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:45,419 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49652, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:15:45,420 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@126a012, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:45,420 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c3d4f84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:45,420 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:15:45,420 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:15:45,421 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:15:45,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:15:45,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:15:45,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3baeede8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:45,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:15:45,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:15:45,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:45,422 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49670, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:15:45,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24b5cab4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:45,423 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ad3f2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:45,423 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:15:45,423 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:15:45,424 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:15:45,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:15:45,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:15:45,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6aec5cb4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:45,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:15:45,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:15:45,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:45,425 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49690, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:15:45,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6383b6bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:45,426 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@422fcba0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:45,426 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:15:45,426 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:15:45,427 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:15:45,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:15:45,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:15:45,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4efe5281, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:45,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:15:45,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:15:45,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:45,428 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49718, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:15:45,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@223fb778, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:45,429 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72e71a40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:45,429 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:15:45,429 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:15:45,429 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:15:45,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:15:45,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:15:45,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ea6ae97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:45,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:15:45,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:15:45,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:45,430 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49736, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:15:45,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2da7f431, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:45,431 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61b758c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:45,431 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:15:45,431 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:15:45,432 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:15:45,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:15:45,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:15:45,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3db5de99, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:45,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:15:45,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:15:45,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:45,433 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49754, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:15:45,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e59f4dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:45,434 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10e2b5a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:45,434 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:15:45,434 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:15:45,435 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:15:45,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:15:45,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:15:45,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dccf358, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:45,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:15:45,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:15:45,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:45,436 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49772, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:15:45,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6007f1d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:45,437 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d41a701, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:45,437 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:15:45,437 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:15:45,438 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:15:45,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:15:45,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:15:45,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c76ddb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:45,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:15:45,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:15:45,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:45,439 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:15:45,439 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5feb55c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:45,440 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b2f46ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:45,440 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:15:45,440 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:15:45,441 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:15:45,441 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:15:45,441 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:15:45,441 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1899887c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:45,441 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:15:45,441 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:15:45,441 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:45,442 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49802, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:15:45,443 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16512fbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:45,443 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@180bd544, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:45,444 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:15:45,444 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:15:45,444 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:15:45,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:15:45,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:15:45,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7134ecb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:45,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:15:45,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:15:45,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:15:45,446 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49830, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:15:45,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@732b91f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:15:45,447 DEBUG [Thread-1619 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:15:45,447 DEBUG [Thread-1625 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:15:45,447 DEBUG [Thread-1610 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:15:45,447 DEBUG [Thread-1627 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:15:45,447 DEBUG [Thread-1623 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:15:45,447 DEBUG [Thread-1621 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:15:45,448 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:15:45,448 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:15:45,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:15:45,448 DEBUG [Thread-1619 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:15:45,448 DEBUG [Thread-1623 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:15:45,448 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:15:45,448 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:15:45,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:15:45,448 DEBUG [Thread-1619 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:15:45,448 DEBUG [Thread-1623 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:15:45,449 DEBUG [Thread-1612 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:15:45,449 DEBUG [Thread-1610 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:15:45,450 DEBUG [Thread-1610 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:15:45,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:15:45,450 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44268, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:15:45,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:15:45,450 DEBUG [Thread-1608 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:15:45,450 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44266, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:15:45,450 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44288, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:15:45,450 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:15:45,450 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44264, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:15:45,451 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44290, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:15:45,451 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44292, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:15:45,451 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:15:45,451 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:15:45,451 DEBUG [Thread-1614 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:15:45,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:15:45,452 DEBUG [Thread-1616 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:15:45,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:15:45,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:15:45,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:15:45,453 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44306, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:15:45,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5] 2024-11-21T21:15:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5] 2024-11-21T21:15:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5] 2024-11-21T21:15:45,455 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:15:45,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5] 2024-11-21T21:15:45,455 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44318, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:15:45,456 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44320, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:15:45,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5] 2024-11-21T21:15:45,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5] 2024-11-21T21:15:45,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5] 2024-11-21T21:15:45,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-21T21:15:45,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5] 2024-11-21T21:15:45,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5] 2024-11-21T21:15:45,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-21T21:15:45,459 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:15:45,462 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:15:45,462 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:15:45,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5] 2024-11-21T21:15:45,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:45,467 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca977aeb13804530742b16bc2c00bf08 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T21:15:45,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=A 2024-11-21T21:15:45,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:45,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=B 2024-11-21T21:15:45,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:45,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=C 2024-11-21T21:15:45,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:45,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411219b8bebd5d8ee471fbe5a9e764f980b68_ca977aeb13804530742b16bc2c00bf08 is 50, key is test_row_0/A:col10/1732223745456/Put/seqid=0 2024-11-21T21:15:45,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742180_1356 (size=14594) 2024-11-21T21:15:45,526 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:15:45,531 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732223745506, max timestamp=1732223745528Num uniques 859; 2024-11-21T21:15:45,531 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T21:15:45,533 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411219b8bebd5d8ee471fbe5a9e764f980b68_ca977aeb13804530742b16bc2c00bf08 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219b8bebd5d8ee471fbe5a9e764f980b68_ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:45,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732223745506, max timestamp=1732223745528Num uniques 859; 2024-11-21T21:15:45,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732223745506, max timestamp=1732223745528Num uniques 859; 2024-11-21T21:15:45,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T21:15:45,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T21:15:45,535 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/b0e262af4e5543b08fbeccb586f3098c, store: [table=TestAcidGuarantees family=A region=ca977aeb13804530742b16bc2c00bf08] 2024-11-21T21:15:45,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/b0e262af4e5543b08fbeccb586f3098c is 175, key is test_row_0/A:col10/1732223745456/Put/seqid=0 2024-11-21T21:15:45,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742181_1357 (size=39549) 2024-11-21T21:15:45,543 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/b0e262af4e5543b08fbeccb586f3098c 2024-11-21T21:15:45,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-21T21:15:45,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/0842509663f843ecba27e2f1a1a2dccd is 50, key is test_row_0/B:col10/1732223745456/Put/seqid=0 2024-11-21T21:15:45,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:45,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44320 deadline: 1732223805578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:45,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:45,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44292 deadline: 1732223805580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:45,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:45,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:45,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:45,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:45,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:45,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:45,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742182_1358 (size=12001) 2024-11-21T21:15:45,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:45,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44306 deadline: 1732223805593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:45,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:45,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:45,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:45,608 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/0842509663f843ecba27e2f1a1a2dccd 2024-11-21T21:15:45,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:45,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44290 deadline: 1732223805596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:45,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:45,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:45,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:45,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-21T21:15:45,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:45,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. as already flushing 2024-11-21T21:15:45,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:45,616 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:45,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:45,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:45,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44318 deadline: 1732223805598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:45,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:45,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:45,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:45,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:45,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/8ae475fe3bc74844a1986f2d823dd505 is 50, key is test_row_0/C:col10/1732223745456/Put/seqid=0 2024-11-21T21:15:45,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742183_1359 (size=12001) 2024-11-21T21:15:45,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:45,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44320 deadline: 1732223805700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:45,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:45,707 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:45,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44292 deadline: 1732223805700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:45,707 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:45,707 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:45,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:45,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:45,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:45,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:45,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44306 deadline: 1732223805710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:45,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:45,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:45,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:45,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:45,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44290 deadline: 1732223805721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:45,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:45,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:45,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:45,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:45,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44318 deadline: 1732223805721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:45,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:45,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:45,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:45,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-21T21:15:45,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:45,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. as already flushing 2024-11-21T21:15:45,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:45,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:45,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:45,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:45,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-21T21:15:45,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:45,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44320 deadline: 1732223805910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:45,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:45,915 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:45,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44292 deadline: 1732223805910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:45,915 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:45,915 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:45,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:45,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:45,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:45,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-21T21:15:45,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:45,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. as already flushing 2024-11-21T21:15:45,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:45,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:45,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:45,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:45,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:45,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44306 deadline: 1732223805930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:45,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:45,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:45,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:45,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:45,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44290 deadline: 1732223805930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:45,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:45,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:45,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44318 deadline: 1732223805930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:46,048 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/8ae475fe3bc74844a1986f2d823dd505 2024-11-21T21:15:46,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/b0e262af4e5543b08fbeccb586f3098c as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/b0e262af4e5543b08fbeccb586f3098c 2024-11-21T21:15:46,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/b0e262af4e5543b08fbeccb586f3098c, entries=200, sequenceid=15, filesize=38.6 K 2024-11-21T21:15:46,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/0842509663f843ecba27e2f1a1a2dccd as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/0842509663f843ecba27e2f1a1a2dccd 2024-11-21T21:15:46,059 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/0842509663f843ecba27e2f1a1a2dccd, entries=150, sequenceid=15, filesize=11.7 K 2024-11-21T21:15:46,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/8ae475fe3bc74844a1986f2d823dd505 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/8ae475fe3bc74844a1986f2d823dd505 2024-11-21T21:15:46,063 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/8ae475fe3bc74844a1986f2d823dd505, entries=150, sequenceid=15, filesize=11.7 K 2024-11-21T21:15:46,064 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for ca977aeb13804530742b16bc2c00bf08 in 596ms, sequenceid=15, compaction requested=false 2024-11-21T21:15:46,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca977aeb13804530742b16bc2c00bf08: 2024-11-21T21:15:46,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-21T21:15:46,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:46,074 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing ca977aeb13804530742b16bc2c00bf08 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-21T21:15:46,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=A 2024-11-21T21:15:46,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:15:46,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=B 2024-11-21T21:15:46,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:15:46,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=C 2024-11-21T21:15:46,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:15:46,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-21T21:15:46,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112145a115508d66474893b3e7d331db6468_ca977aeb13804530742b16bc2c00bf08 is 50, key is test_row_0/A:col10/1732223745593/Put/seqid=0 2024-11-21T21:15:46,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742184_1360 (size=12154) 2024-11-21T21:15:46,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:46,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. as already flushing 2024-11-21T21:15:46,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:46,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44292 deadline: 1732223806246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:46,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:46,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:46,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:46,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:46,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44320 deadline: 1732223806247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:46,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:46,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:46,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44318 deadline: 1732223806248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:46,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:46,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:46,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:46,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:46,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:46,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:46,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44306 deadline: 1732223806250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:46,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:46,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:46,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:46,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:46,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44290 deadline: 1732223806250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:46,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:46,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:46,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:46,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:46,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44292 deadline: 1732223806360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:46,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:46,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:46,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:46,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:46,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44320 deadline: 1732223806370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:46,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:46,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:46,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44318 deadline: 1732223806370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:46,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:46,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:46,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:46,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:46,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:46,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:15:46,539 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112145a115508d66474893b3e7d331db6468_ca977aeb13804530742b16bc2c00bf08 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112145a115508d66474893b3e7d331db6468_ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:46,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/d6b4d05725ee437a98c7ae6d5440f7af, store: [table=TestAcidGuarantees family=A region=ca977aeb13804530742b16bc2c00bf08] 2024-11-21T21:15:46,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/d6b4d05725ee437a98c7ae6d5440f7af is 175, key is test_row_0/A:col10/1732223745593/Put/seqid=0 2024-11-21T21:15:46,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742185_1361 (size=30955) 2024-11-21T21:15:46,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:46,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44292 deadline: 1732223806580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:46,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:46,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:46,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:46,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44320 deadline: 1732223806580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:46,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-21T21:15:46,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:46,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44318 deadline: 1732223806590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:46,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:46,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:46,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:46,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:46,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44306 deadline: 1732223806770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:46,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:46,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:46,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44290 deadline: 1732223806770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:46,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:46,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:46,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:46,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:46,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:46,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:46,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44292 deadline: 1732223806890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:46,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:46,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:46,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:46,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44320 deadline: 1732223806890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:46,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:46,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:46,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:46,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:46,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:46,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44318 deadline: 1732223806910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:46,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:46,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:46,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:46,948 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/d6b4d05725ee437a98c7ae6d5440f7af 2024-11-21T21:15:46,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/412f378a46eb45cb8be34e24420073b6 is 50, key is test_row_0/B:col10/1732223745593/Put/seqid=0 2024-11-21T21:15:46,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742186_1362 (size=12001) 2024-11-21T21:15:47,360 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/412f378a46eb45cb8be34e24420073b6 2024-11-21T21:15:47,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/d125e4864f514ce19fcecb4078cf2fd3 is 50, key is test_row_0/C:col10/1732223745593/Put/seqid=0 2024-11-21T21:15:47,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742187_1363 (size=12001) 2024-11-21T21:15:47,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:47,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44292 deadline: 1732223807410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:47,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:47,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44320 deadline: 1732223807410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:47,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:47,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:47,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:47,415 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:47,415 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:47,415 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:47,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:47,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44318 deadline: 1732223807420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:47,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:47,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:47,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:47,570 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T21:15:47,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-21T21:15:47,774 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/d125e4864f514ce19fcecb4078cf2fd3 2024-11-21T21:15:47,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/d6b4d05725ee437a98c7ae6d5440f7af as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/d6b4d05725ee437a98c7ae6d5440f7af 2024-11-21T21:15:47,781 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/d6b4d05725ee437a98c7ae6d5440f7af, entries=150, sequenceid=50, filesize=30.2 K 2024-11-21T21:15:47,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/412f378a46eb45cb8be34e24420073b6 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/412f378a46eb45cb8be34e24420073b6 2024-11-21T21:15:47,785 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/412f378a46eb45cb8be34e24420073b6, entries=150, sequenceid=50, filesize=11.7 K 2024-11-21T21:15:47,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/d125e4864f514ce19fcecb4078cf2fd3 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/d125e4864f514ce19fcecb4078cf2fd3 2024-11-21T21:15:47,789 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/d125e4864f514ce19fcecb4078cf2fd3, entries=150, sequenceid=50, filesize=11.7 K 2024-11-21T21:15:47,790 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for ca977aeb13804530742b16bc2c00bf08 in 1716ms, sequenceid=50, compaction requested=false 2024-11-21T21:15:47,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for ca977aeb13804530742b16bc2c00bf08: 2024-11-21T21:15:47,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:47,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-21T21:15:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-21T21:15:47,793 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-21T21:15:47,793 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3290 sec 2024-11-21T21:15:47,794 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.3380 sec 2024-11-21T21:15:47,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:47,795 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca977aeb13804530742b16bc2c00bf08 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-21T21:15:47,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=A 2024-11-21T21:15:47,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:47,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=B 2024-11-21T21:15:47,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:47,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=C 2024-11-21T21:15:47,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:47,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112153821b4bdfc142958564f49a00d2b3bb_ca977aeb13804530742b16bc2c00bf08 is 50, key is test_row_0/A:col10/1732223746246/Put/seqid=0 2024-11-21T21:15:47,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742188_1364 (size=14594) 2024-11-21T21:15:47,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:47,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44290 deadline: 1732223807874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:47,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:47,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:47,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44306 deadline: 1732223807875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:47,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:47,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:47,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:47,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:47,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:47,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:47,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44290 deadline: 1732223807990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:47,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:47,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:47,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:47,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:47,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44306 deadline: 1732223807990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:47,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:47,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:47,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:48,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:48,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44290 deadline: 1732223808201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:48,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:48,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:48,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:48,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:48,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44306 deadline: 1732223808201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:48,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:48,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:48,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:48,210 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:15:48,213 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112153821b4bdfc142958564f49a00d2b3bb_ca977aeb13804530742b16bc2c00bf08 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112153821b4bdfc142958564f49a00d2b3bb_ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:48,214 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/a7832558992d4ff3a431c952b6003571, store: [table=TestAcidGuarantees family=A region=ca977aeb13804530742b16bc2c00bf08] 2024-11-21T21:15:48,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/a7832558992d4ff3a431c952b6003571 is 175, key is test_row_0/A:col10/1732223746246/Put/seqid=0 2024-11-21T21:15:48,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742189_1365 (size=39549) 2024-11-21T21:15:48,224 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/a7832558992d4ff3a431c952b6003571 2024-11-21T21:15:48,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/02c5d888bd014d93be1b1e632d1d55b9 is 50, key is test_row_0/B:col10/1732223746246/Put/seqid=0 2024-11-21T21:15:48,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742190_1366 (size=12001) 2024-11-21T21:15:48,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:48,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44292 deadline: 1732223808420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:48,429 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,41213,1732223653469 for 'test_row_2' in ca977aeb13804530742b16bc2c00bf08 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:15:48,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:48,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:48,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:48,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:48,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44320 deadline: 1732223808430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:48,440 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,41213,1732223653469 for 'test_row_1' in ca977aeb13804530742b16bc2c00bf08 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:15:48,441 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:48,441 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:48,441 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:48,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:48,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44318 deadline: 1732223808440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:48,443 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,41213,1732223653469 for 'test_row_1' in ca977aeb13804530742b16bc2c00bf08 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:15:48,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:48,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:48,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:48,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:48,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44290 deadline: 1732223808510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:48,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:48,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:48,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:48,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44306 deadline: 1732223808510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:48,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:48,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:48,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:48,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:48,637 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/02c5d888bd014d93be1b1e632d1d55b9 2024-11-21T21:15:48,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/8edb86085f614ff08e92a374963150e7 is 50, key is test_row_0/C:col10/1732223746246/Put/seqid=0 2024-11-21T21:15:48,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742191_1367 (size=12001) 2024-11-21T21:15:49,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:49,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44290 deadline: 1732223809030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:49,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:49,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44306 deadline: 1732223809030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:49,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:49,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:49,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:49,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:49,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:49,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:49,055 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/8edb86085f614ff08e92a374963150e7 2024-11-21T21:15:49,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/a7832558992d4ff3a431c952b6003571 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/a7832558992d4ff3a431c952b6003571 2024-11-21T21:15:49,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/a7832558992d4ff3a431c952b6003571, entries=200, sequenceid=63, filesize=38.6 K 2024-11-21T21:15:49,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/02c5d888bd014d93be1b1e632d1d55b9 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/02c5d888bd014d93be1b1e632d1d55b9 2024-11-21T21:15:49,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/02c5d888bd014d93be1b1e632d1d55b9, entries=150, sequenceid=63, filesize=11.7 K 2024-11-21T21:15:49,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/8edb86085f614ff08e92a374963150e7 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/8edb86085f614ff08e92a374963150e7 2024-11-21T21:15:49,067 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/8edb86085f614ff08e92a374963150e7, entries=150, sequenceid=63, filesize=11.7 K 2024-11-21T21:15:49,068 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for ca977aeb13804530742b16bc2c00bf08 in 1273ms, sequenceid=63, compaction requested=true 2024-11-21T21:15:49,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca977aeb13804530742b16bc2c00bf08: 2024-11-21T21:15:49,068 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:15:49,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca977aeb13804530742b16bc2c00bf08:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:15:49,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:49,068 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:15:49,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca977aeb13804530742b16bc2c00bf08:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:15:49,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:49,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca977aeb13804530742b16bc2c00bf08:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:15:49,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:15:49,069 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:15:49,069 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): ca977aeb13804530742b16bc2c00bf08/A is initiating minor compaction (all files) 2024-11-21T21:15:49,069 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca977aeb13804530742b16bc2c00bf08/A in TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:49,069 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/b0e262af4e5543b08fbeccb586f3098c, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/d6b4d05725ee437a98c7ae6d5440f7af, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/a7832558992d4ff3a431c952b6003571] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp, totalSize=107.5 K 2024-11-21T21:15:49,069 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:49,069 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. files: [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/b0e262af4e5543b08fbeccb586f3098c, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/d6b4d05725ee437a98c7ae6d5440f7af, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/a7832558992d4ff3a431c952b6003571] 2024-11-21T21:15:49,070 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:15:49,070 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): ca977aeb13804530742b16bc2c00bf08/B is initiating minor compaction (all files) 2024-11-21T21:15:49,070 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0e262af4e5543b08fbeccb586f3098c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732223745456 2024-11-21T21:15:49,070 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca977aeb13804530742b16bc2c00bf08/B in TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:49,070 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/0842509663f843ecba27e2f1a1a2dccd, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/412f378a46eb45cb8be34e24420073b6, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/02c5d888bd014d93be1b1e632d1d55b9] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp, totalSize=35.2 K 2024-11-21T21:15:49,070 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6b4d05725ee437a98c7ae6d5440f7af, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732223745577 2024-11-21T21:15:49,070 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 0842509663f843ecba27e2f1a1a2dccd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732223745456 2024-11-21T21:15:49,071 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7832558992d4ff3a431c952b6003571, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732223746229 2024-11-21T21:15:49,071 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 412f378a46eb45cb8be34e24420073b6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732223745577 2024-11-21T21:15:49,071 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 02c5d888bd014d93be1b1e632d1d55b9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732223746246 2024-11-21T21:15:49,075 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ca977aeb13804530742b16bc2c00bf08] 2024-11-21T21:15:49,076 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca977aeb13804530742b16bc2c00bf08#B#compaction#323 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:49,077 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/2e4c80e7a16f4408b8a7f643485ca7ca is 50, key is test_row_0/B:col10/1732223746246/Put/seqid=0 2024-11-21T21:15:49,077 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121c266e0c9e28d4b27ad7f54e91107b037_ca977aeb13804530742b16bc2c00bf08 store=[table=TestAcidGuarantees family=A region=ca977aeb13804530742b16bc2c00bf08] 2024-11-21T21:15:49,080 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121c266e0c9e28d4b27ad7f54e91107b037_ca977aeb13804530742b16bc2c00bf08, store=[table=TestAcidGuarantees family=A region=ca977aeb13804530742b16bc2c00bf08] 2024-11-21T21:15:49,080 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c266e0c9e28d4b27ad7f54e91107b037_ca977aeb13804530742b16bc2c00bf08 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ca977aeb13804530742b16bc2c00bf08] 2024-11-21T21:15:49,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742192_1368 (size=12104) 2024-11-21T21:15:49,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742193_1369 (size=4469) 2024-11-21T21:15:49,097 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/2e4c80e7a16f4408b8a7f643485ca7ca as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/2e4c80e7a16f4408b8a7f643485ca7ca 2024-11-21T21:15:49,104 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca977aeb13804530742b16bc2c00bf08/B of ca977aeb13804530742b16bc2c00bf08 into 2e4c80e7a16f4408b8a7f643485ca7ca(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:49,104 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca977aeb13804530742b16bc2c00bf08: 2024-11-21T21:15:49,104 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., storeName=ca977aeb13804530742b16bc2c00bf08/B, priority=13, startTime=1732223749068; duration=0sec 2024-11-21T21:15:49,104 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:15:49,104 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca977aeb13804530742b16bc2c00bf08:B 2024-11-21T21:15:49,104 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:15:49,111 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:15:49,111 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): ca977aeb13804530742b16bc2c00bf08/C is initiating minor compaction (all files) 2024-11-21T21:15:49,111 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca977aeb13804530742b16bc2c00bf08/C in TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:49,111 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/8ae475fe3bc74844a1986f2d823dd505, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/d125e4864f514ce19fcecb4078cf2fd3, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/8edb86085f614ff08e92a374963150e7] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp, totalSize=35.2 K 2024-11-21T21:15:49,112 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ae475fe3bc74844a1986f2d823dd505, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732223745456 2024-11-21T21:15:49,112 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting d125e4864f514ce19fcecb4078cf2fd3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732223745577 2024-11-21T21:15:49,113 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 8edb86085f614ff08e92a374963150e7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732223746246 2024-11-21T21:15:49,122 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca977aeb13804530742b16bc2c00bf08#C#compaction#325 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:49,122 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/eb1b40de11db48829af46ad3e5fbc1fa is 50, key is test_row_0/C:col10/1732223746246/Put/seqid=0 2024-11-21T21:15:49,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742194_1370 (size=12104) 2024-11-21T21:15:49,494 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca977aeb13804530742b16bc2c00bf08#A#compaction#324 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:49,494 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/59c59dd39e9e4ffabfd25aeea9fcc6f8 is 175, key is test_row_0/A:col10/1732223746246/Put/seqid=0 2024-11-21T21:15:49,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742195_1371 (size=31058) 2024-11-21T21:15:49,529 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/eb1b40de11db48829af46ad3e5fbc1fa as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/eb1b40de11db48829af46ad3e5fbc1fa 2024-11-21T21:15:49,533 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca977aeb13804530742b16bc2c00bf08/C of ca977aeb13804530742b16bc2c00bf08 into eb1b40de11db48829af46ad3e5fbc1fa(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:49,533 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca977aeb13804530742b16bc2c00bf08: 2024-11-21T21:15:49,533 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., storeName=ca977aeb13804530742b16bc2c00bf08/C, priority=13, startTime=1732223749069; duration=0sec 2024-11-21T21:15:49,533 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:49,533 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca977aeb13804530742b16bc2c00bf08:C 2024-11-21T21:15:49,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-21T21:15:49,611 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:15:49,611 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:15:49,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-21T21:15:49,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-21T21:15:49,613 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:15:49,613 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:15:49,613 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:15:49,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-21T21:15:49,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-21T21:15:49,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:49,766 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing ca977aeb13804530742b16bc2c00bf08 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-21T21:15:49,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=A 2024-11-21T21:15:49,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:49,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=B 2024-11-21T21:15:49,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:49,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=C 2024-11-21T21:15:49,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:49,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c622d47bfe2a4fe4b648863f89247c1a_ca977aeb13804530742b16bc2c00bf08 is 50, key is test_row_0/A:col10/1732223747834/Put/seqid=0 2024-11-21T21:15:49,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742196_1372 (size=12154) 2024-11-21T21:15:49,902 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/59c59dd39e9e4ffabfd25aeea9fcc6f8 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/59c59dd39e9e4ffabfd25aeea9fcc6f8 2024-11-21T21:15:49,906 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca977aeb13804530742b16bc2c00bf08/A of ca977aeb13804530742b16bc2c00bf08 into 59c59dd39e9e4ffabfd25aeea9fcc6f8(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:49,906 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca977aeb13804530742b16bc2c00bf08: 2024-11-21T21:15:49,906 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., storeName=ca977aeb13804530742b16bc2c00bf08/A, priority=13, startTime=1732223749068; duration=0sec 2024-11-21T21:15:49,906 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:49,906 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca977aeb13804530742b16bc2c00bf08:A 2024-11-21T21:15:49,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-21T21:15:50,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:50,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. as already flushing 2024-11-21T21:15:50,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:50,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44290 deadline: 1732223810070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:50,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:50,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:50,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:50,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:50,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44306 deadline: 1732223810077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:50,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:50,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:50,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:50,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:15:50,178 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c622d47bfe2a4fe4b648863f89247c1a_ca977aeb13804530742b16bc2c00bf08 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c622d47bfe2a4fe4b648863f89247c1a_ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:50,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/96c371fbe67d43d2bd68d6523c205231, store: [table=TestAcidGuarantees family=A region=ca977aeb13804530742b16bc2c00bf08] 2024-11-21T21:15:50,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/96c371fbe67d43d2bd68d6523c205231 is 175, key is test_row_0/A:col10/1732223747834/Put/seqid=0 2024-11-21T21:15:50,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742197_1373 (size=30955) 2024-11-21T21:15:50,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:50,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44290 deadline: 1732223810181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:50,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:50,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:50,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:50,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:50,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44306 deadline: 1732223810190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:50,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:50,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:50,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:50,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-21T21:15:50,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:50,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44290 deadline: 1732223810400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:50,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:50,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:50,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:50,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44306 deadline: 1732223810401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:50,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:50,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:50,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:50,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:50,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:50,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44292 deadline: 1732223810450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:50,458 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,41213,1732223653469 for 'test_row_2' in ca977aeb13804530742b16bc2c00bf08 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:15:50,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:50,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:50,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:50,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:50,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44318 deadline: 1732223810460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:50,470 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,41213,1732223653469 for 'test_row_1' in ca977aeb13804530742b16bc2c00bf08 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:15:50,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:50,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:50,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:50,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:50,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44320 deadline: 1732223810471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:50,475 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,41213,1732223653469 for 'test_row_1' in ca977aeb13804530742b16bc2c00bf08 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:15:50,476 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:50,476 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:50,476 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:50,582 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=85, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/96c371fbe67d43d2bd68d6523c205231 2024-11-21T21:15:50,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/69acb7225da14e04bbd16e08e00bd894 is 50, key is test_row_0/B:col10/1732223747834/Put/seqid=0 2024-11-21T21:15:50,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742198_1374 (size=12001) 2024-11-21T21:15:50,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:50,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44306 deadline: 1732223810710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:50,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:50,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:50,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:50,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:50,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44290 deadline: 1732223810711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:50,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:50,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:50,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:50,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-21T21:15:50,995 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/69acb7225da14e04bbd16e08e00bd894 2024-11-21T21:15:51,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/45b3fb64d7a144c4917a990a9276b779 is 50, key is test_row_0/C:col10/1732223747834/Put/seqid=0 2024-11-21T21:15:51,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742199_1375 (size=12001) 2024-11-21T21:15:51,006 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/45b3fb64d7a144c4917a990a9276b779 2024-11-21T21:15:51,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/96c371fbe67d43d2bd68d6523c205231 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/96c371fbe67d43d2bd68d6523c205231 2024-11-21T21:15:51,016 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/96c371fbe67d43d2bd68d6523c205231, entries=150, sequenceid=85, filesize=30.2 K 2024-11-21T21:15:51,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/69acb7225da14e04bbd16e08e00bd894 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/69acb7225da14e04bbd16e08e00bd894 2024-11-21T21:15:51,022 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/69acb7225da14e04bbd16e08e00bd894, entries=150, sequenceid=85, filesize=11.7 K 2024-11-21T21:15:51,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/45b3fb64d7a144c4917a990a9276b779 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/45b3fb64d7a144c4917a990a9276b779 2024-11-21T21:15:51,026 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/45b3fb64d7a144c4917a990a9276b779, entries=150, sequenceid=85, filesize=11.7 K 2024-11-21T21:15:51,027 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for ca977aeb13804530742b16bc2c00bf08 in 1261ms, sequenceid=85, compaction requested=false 2024-11-21T21:15:51,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for ca977aeb13804530742b16bc2c00bf08: 2024-11-21T21:15:51,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:51,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-21T21:15:51,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-21T21:15:51,032 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-21T21:15:51,032 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4170 sec 2024-11-21T21:15:51,034 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.4210 sec 2024-11-21T21:15:51,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:51,223 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca977aeb13804530742b16bc2c00bf08 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-21T21:15:51,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=A 2024-11-21T21:15:51,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:51,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=B 2024-11-21T21:15:51,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:51,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=C 2024-11-21T21:15:51,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:51,231 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411211759110c719f4a15aaf53424769ca8ad_ca977aeb13804530742b16bc2c00bf08 is 50, key is test_row_0/A:col10/1732223750075/Put/seqid=0 2024-11-21T21:15:51,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742200_1376 (size=14594) 2024-11-21T21:15:51,244 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:15:51,250 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411211759110c719f4a15aaf53424769ca8ad_ca977aeb13804530742b16bc2c00bf08 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411211759110c719f4a15aaf53424769ca8ad_ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:51,251 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/5860f266f3854e90857c5d46008fcd9f, store: [table=TestAcidGuarantees family=A region=ca977aeb13804530742b16bc2c00bf08] 2024-11-21T21:15:51,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/5860f266f3854e90857c5d46008fcd9f is 175, key is test_row_0/A:col10/1732223750075/Put/seqid=0 2024-11-21T21:15:51,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742201_1377 (size=39549) 2024-11-21T21:15:51,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:51,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44306 deadline: 1732223811294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:51,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:51,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:51,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:51,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:51,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44290 deadline: 1732223811300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:51,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:51,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:51,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:51,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:51,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44306 deadline: 1732223811410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:51,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:51,417 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:51,417 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:51,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44290 deadline: 1732223811411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:51,417 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:51,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:51,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:51,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:51,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44290 deadline: 1732223811620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:51,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:51,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:51,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:51,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:51,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44306 deadline: 1732223811631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:51,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:51,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:51,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:51,658 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=99, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/5860f266f3854e90857c5d46008fcd9f 2024-11-21T21:15:51,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/0f345d9862154220953bd9749d59137e is 50, key is test_row_0/B:col10/1732223750075/Put/seqid=0 2024-11-21T21:15:51,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742202_1378 (size=12001) 2024-11-21T21:15:51,712 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/0f345d9862154220953bd9749d59137e 2024-11-21T21:15:51,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/513934083c8a493baa05f0697fe636e1 is 50, key is test_row_0/C:col10/1732223750075/Put/seqid=0 2024-11-21T21:15:51,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742203_1379 (size=12001) 2024-11-21T21:15:51,738 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/513934083c8a493baa05f0697fe636e1 2024-11-21T21:15:51,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/5860f266f3854e90857c5d46008fcd9f as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/5860f266f3854e90857c5d46008fcd9f 2024-11-21T21:15:51,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/5860f266f3854e90857c5d46008fcd9f, entries=200, sequenceid=99, filesize=38.6 K 2024-11-21T21:15:51,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/0f345d9862154220953bd9749d59137e as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/0f345d9862154220953bd9749d59137e 2024-11-21T21:15:51,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/0f345d9862154220953bd9749d59137e, entries=150, sequenceid=99, filesize=11.7 K 2024-11-21T21:15:51,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/513934083c8a493baa05f0697fe636e1 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/513934083c8a493baa05f0697fe636e1 2024-11-21T21:15:51,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-21T21:15:51,760 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:15:51,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/513934083c8a493baa05f0697fe636e1, entries=150, sequenceid=99, filesize=11.7 K 2024-11-21T21:15:51,761 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:15:51,761 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for ca977aeb13804530742b16bc2c00bf08 in 538ms, sequenceid=99, compaction requested=true 2024-11-21T21:15:51,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca977aeb13804530742b16bc2c00bf08: 2024-11-21T21:15:51,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-21T21:15:51,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca977aeb13804530742b16bc2c00bf08:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:15:51,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:51,762 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:15:51,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca977aeb13804530742b16bc2c00bf08:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:15:51,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:51,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca977aeb13804530742b16bc2c00bf08:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:15:51,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:15:51,762 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:15:51,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-21T21:15:51,762 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:15:51,763 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:15:51,763 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:15:51,763 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): ca977aeb13804530742b16bc2c00bf08/B is initiating minor compaction (all files) 2024-11-21T21:15:51,763 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): ca977aeb13804530742b16bc2c00bf08/A is initiating minor compaction (all files) 2024-11-21T21:15:51,763 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca977aeb13804530742b16bc2c00bf08/B in TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:51,763 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca977aeb13804530742b16bc2c00bf08/A in TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:51,763 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/2e4c80e7a16f4408b8a7f643485ca7ca, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/69acb7225da14e04bbd16e08e00bd894, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/0f345d9862154220953bd9749d59137e] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp, totalSize=35.3 K 2024-11-21T21:15:51,763 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/59c59dd39e9e4ffabfd25aeea9fcc6f8, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/96c371fbe67d43d2bd68d6523c205231, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/5860f266f3854e90857c5d46008fcd9f] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp, totalSize=99.2 K 2024-11-21T21:15:51,763 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:51,763 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. files: [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/59c59dd39e9e4ffabfd25aeea9fcc6f8, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/96c371fbe67d43d2bd68d6523c205231, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/5860f266f3854e90857c5d46008fcd9f] 2024-11-21T21:15:51,763 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e4c80e7a16f4408b8a7f643485ca7ca, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732223746246 2024-11-21T21:15:51,764 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:15:51,764 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:15:51,764 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 69acb7225da14e04bbd16e08e00bd894, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1732223747834 2024-11-21T21:15:51,764 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f345d9862154220953bd9749d59137e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732223750069 2024-11-21T21:15:51,766 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59c59dd39e9e4ffabfd25aeea9fcc6f8, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732223746246 2024-11-21T21:15:51,767 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96c371fbe67d43d2bd68d6523c205231, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1732223747834 2024-11-21T21:15:51,767 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5860f266f3854e90857c5d46008fcd9f, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732223750061 2024-11-21T21:15:51,775 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca977aeb13804530742b16bc2c00bf08#B#compaction#332 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:51,776 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/74efbfadb8e745129f8539369ba18b1f is 50, key is test_row_0/B:col10/1732223750075/Put/seqid=0 2024-11-21T21:15:51,790 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ca977aeb13804530742b16bc2c00bf08] 2024-11-21T21:15:51,797 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121cd7eab5dc98f4d75ac64202f1cf9aa9a_ca977aeb13804530742b16bc2c00bf08 store=[table=TestAcidGuarantees family=A region=ca977aeb13804530742b16bc2c00bf08] 2024-11-21T21:15:51,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742204_1380 (size=12207) 2024-11-21T21:15:51,800 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121cd7eab5dc98f4d75ac64202f1cf9aa9a_ca977aeb13804530742b16bc2c00bf08, store=[table=TestAcidGuarantees family=A region=ca977aeb13804530742b16bc2c00bf08] 2024-11-21T21:15:51,800 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121cd7eab5dc98f4d75ac64202f1cf9aa9a_ca977aeb13804530742b16bc2c00bf08 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ca977aeb13804530742b16bc2c00bf08] 2024-11-21T21:15:51,814 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/74efbfadb8e745129f8539369ba18b1f as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/74efbfadb8e745129f8539369ba18b1f 2024-11-21T21:15:51,820 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca977aeb13804530742b16bc2c00bf08/B of ca977aeb13804530742b16bc2c00bf08 into 74efbfadb8e745129f8539369ba18b1f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:51,820 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca977aeb13804530742b16bc2c00bf08: 2024-11-21T21:15:51,820 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., storeName=ca977aeb13804530742b16bc2c00bf08/B, priority=13, startTime=1732223751762; duration=0sec 2024-11-21T21:15:51,820 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:15:51,820 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca977aeb13804530742b16bc2c00bf08:B 2024-11-21T21:15:51,820 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:15:51,825 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:15:51,825 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): ca977aeb13804530742b16bc2c00bf08/C is initiating minor compaction (all files) 2024-11-21T21:15:51,825 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca977aeb13804530742b16bc2c00bf08/C in TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:51,825 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/eb1b40de11db48829af46ad3e5fbc1fa, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/45b3fb64d7a144c4917a990a9276b779, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/513934083c8a493baa05f0697fe636e1] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp, totalSize=35.3 K 2024-11-21T21:15:51,825 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting eb1b40de11db48829af46ad3e5fbc1fa, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732223746246 2024-11-21T21:15:51,826 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 45b3fb64d7a144c4917a990a9276b779, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1732223747834 2024-11-21T21:15:51,826 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 513934083c8a493baa05f0697fe636e1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732223750069 2024-11-21T21:15:51,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742205_1381 (size=4469) 2024-11-21T21:15:51,834 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca977aeb13804530742b16bc2c00bf08#A#compaction#333 average throughput is 0.56 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:51,835 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/25b44c4d01744bbc8dbc9eadf9840fb4 is 175, key is test_row_0/A:col10/1732223750075/Put/seqid=0 2024-11-21T21:15:51,838 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca977aeb13804530742b16bc2c00bf08#C#compaction#334 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:51,838 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/1d48266e64874a9d999f5ae05785c765 is 50, key is test_row_0/C:col10/1732223750075/Put/seqid=0 2024-11-21T21:15:51,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-21T21:15:51,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742206_1382 (size=31161) 2024-11-21T21:15:51,882 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/25b44c4d01744bbc8dbc9eadf9840fb4 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/25b44c4d01744bbc8dbc9eadf9840fb4 2024-11-21T21:15:51,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742207_1383 (size=12207) 2024-11-21T21:15:51,890 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca977aeb13804530742b16bc2c00bf08/A of ca977aeb13804530742b16bc2c00bf08 into 25b44c4d01744bbc8dbc9eadf9840fb4(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:51,890 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca977aeb13804530742b16bc2c00bf08: 2024-11-21T21:15:51,890 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., storeName=ca977aeb13804530742b16bc2c00bf08/A, priority=13, startTime=1732223751761; duration=0sec 2024-11-21T21:15:51,891 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:51,891 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca977aeb13804530742b16bc2c00bf08:A 2024-11-21T21:15:51,893 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/1d48266e64874a9d999f5ae05785c765 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/1d48266e64874a9d999f5ae05785c765 2024-11-21T21:15:51,901 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca977aeb13804530742b16bc2c00bf08/C of ca977aeb13804530742b16bc2c00bf08 into 1d48266e64874a9d999f5ae05785c765(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:51,901 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca977aeb13804530742b16bc2c00bf08: 2024-11-21T21:15:51,901 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., storeName=ca977aeb13804530742b16bc2c00bf08/C, priority=13, startTime=1732223751762; duration=0sec 2024-11-21T21:15:51,901 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:51,901 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca977aeb13804530742b16bc2c00bf08:C 2024-11-21T21:15:51,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-21T21:15:51,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:51,917 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing ca977aeb13804530742b16bc2c00bf08 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-21T21:15:51,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=A 2024-11-21T21:15:51,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:51,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=B 2024-11-21T21:15:51,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:51,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=C 2024-11-21T21:15:51,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:51,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:51,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. as already flushing 2024-11-21T21:15:51,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121f70c9629e2aa441db0ee9f5ef921380c_ca977aeb13804530742b16bc2c00bf08 is 50, key is test_row_0/A:col10/1732223751281/Put/seqid=0 2024-11-21T21:15:51,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742208_1384 (size=12154) 2024-11-21T21:15:52,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:52,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44306 deadline: 1732223812012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:52,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:52,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:52,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:52,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:52,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44290 deadline: 1732223812016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:52,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:52,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:52,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:52,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-21T21:15:52,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:52,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44306 deadline: 1732223812120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:52,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:52,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:52,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:52,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:52,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44290 deadline: 1732223812130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:52,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:52,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:52,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:52,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:52,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44306 deadline: 1732223812341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:52,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:52,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:52,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:52,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44290 deadline: 1732223812341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:52,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:52,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:52,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:52,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:52,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:15:52,387 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121f70c9629e2aa441db0ee9f5ef921380c_ca977aeb13804530742b16bc2c00bf08 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121f70c9629e2aa441db0ee9f5ef921380c_ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:52,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/b7f0096fc5aa4c01b6d2cc36de776428, store: [table=TestAcidGuarantees family=A region=ca977aeb13804530742b16bc2c00bf08] 2024-11-21T21:15:52,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/b7f0096fc5aa4c01b6d2cc36de776428 is 175, key is test_row_0/A:col10/1732223751281/Put/seqid=0 2024-11-21T21:15:52,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-21T21:15:52,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742209_1385 (size=30955) 2024-11-21T21:15:52,420 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=121, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/b7f0096fc5aa4c01b6d2cc36de776428 2024-11-21T21:15:52,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/34e6f147e7504862a749912eb7d95e2e is 50, key is test_row_0/B:col10/1732223751281/Put/seqid=0 2024-11-21T21:15:52,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742210_1386 (size=12001) 2024-11-21T21:15:52,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:52,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44306 deadline: 1732223812650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:52,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:52,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44290 deadline: 1732223812651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:52,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:52,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:52,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:52,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:52,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:52,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:52,892 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/34e6f147e7504862a749912eb7d95e2e 2024-11-21T21:15:52,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/6ad36bbe4741445d974a1733f083c8a9 is 50, key is test_row_0/C:col10/1732223751281/Put/seqid=0 2024-11-21T21:15:52,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-21T21:15:52,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742211_1387 (size=12001) 2024-11-21T21:15:52,909 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/6ad36bbe4741445d974a1733f083c8a9 2024-11-21T21:15:52,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/b7f0096fc5aa4c01b6d2cc36de776428 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/b7f0096fc5aa4c01b6d2cc36de776428 2024-11-21T21:15:52,921 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/b7f0096fc5aa4c01b6d2cc36de776428, entries=150, sequenceid=121, filesize=30.2 K 2024-11-21T21:15:52,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/34e6f147e7504862a749912eb7d95e2e as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/34e6f147e7504862a749912eb7d95e2e 2024-11-21T21:15:52,927 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/34e6f147e7504862a749912eb7d95e2e, entries=150, sequenceid=121, filesize=11.7 K 2024-11-21T21:15:52,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/6ad36bbe4741445d974a1733f083c8a9 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/6ad36bbe4741445d974a1733f083c8a9 2024-11-21T21:15:52,938 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/6ad36bbe4741445d974a1733f083c8a9, entries=150, sequenceid=121, filesize=11.7 K 2024-11-21T21:15:52,939 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for ca977aeb13804530742b16bc2c00bf08 in 1022ms, sequenceid=121, compaction requested=false 2024-11-21T21:15:52,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for ca977aeb13804530742b16bc2c00bf08: 2024-11-21T21:15:52,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:52,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-21T21:15:52,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-21T21:15:52,943 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-21T21:15:52,943 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1770 sec 2024-11-21T21:15:52,944 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.1820 sec 2024-11-21T21:15:53,010 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-21T21:15:53,010 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-21T21:15:53,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:53,166 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca977aeb13804530742b16bc2c00bf08 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-21T21:15:53,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=A 2024-11-21T21:15:53,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:53,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=B 2024-11-21T21:15:53,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:53,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=C 2024-11-21T21:15:53,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:53,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411210ed6e12e622a4422ab93745f053e202e_ca977aeb13804530742b16bc2c00bf08 is 50, key is test_row_0/A:col10/1732223753165/Put/seqid=0 2024-11-21T21:15:53,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742212_1388 (size=14794) 2024-11-21T21:15:53,182 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:15:53,186 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411210ed6e12e622a4422ab93745f053e202e_ca977aeb13804530742b16bc2c00bf08 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411210ed6e12e622a4422ab93745f053e202e_ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:53,187 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/4ec38aad492e45b3aba8bdb5a377e1b5, store: [table=TestAcidGuarantees family=A region=ca977aeb13804530742b16bc2c00bf08] 2024-11-21T21:15:53,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/4ec38aad492e45b3aba8bdb5a377e1b5 is 175, key is test_row_0/A:col10/1732223753165/Put/seqid=0 2024-11-21T21:15:53,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742213_1389 (size=39749) 2024-11-21T21:15:53,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:53,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44290 deadline: 1732223813229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:53,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:53,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:53,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:53,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:53,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44306 deadline: 1732223813235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:53,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:53,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:53,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:53,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:53,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44290 deadline: 1732223813340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:53,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:53,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:53,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:53,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:53,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44306 deadline: 1732223813351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:53,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:53,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:53,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:53,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:53,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44290 deadline: 1732223813560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:53,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:53,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:53,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:53,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:53,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44306 deadline: 1732223813560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:53,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:53,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:53,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:53,591 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=135, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/4ec38aad492e45b3aba8bdb5a377e1b5 2024-11-21T21:15:53,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/a3434d3b12c04157a1fab47eb20ffb49 is 50, key is test_row_0/B:col10/1732223753165/Put/seqid=0 2024-11-21T21:15:53,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742214_1390 (size=12151) 2024-11-21T21:15:53,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:53,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44290 deadline: 1732223813870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:53,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:53,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:53,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44306 deadline: 1732223813870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:53,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:53,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:53,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:53,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:53,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:53,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-21T21:15:53,921 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:15:53,921 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:15:53,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-21T21:15:53,923 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:15:53,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T21:15:53,923 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:15:53,923 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:15:54,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/a3434d3b12c04157a1fab47eb20ffb49 2024-11-21T21:15:54,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/19c0269a9ee34b1d8341cda515c53882 is 50, key is test_row_0/C:col10/1732223753165/Put/seqid=0 2024-11-21T21:15:54,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742215_1391 (size=12151) 2024-11-21T21:15:54,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T21:15:54,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T21:15:54,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:54,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. as already flushing 2024-11-21T21:15:54,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:54,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:54,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:54,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:54,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T21:15:54,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:54,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. as already flushing 2024-11-21T21:15:54,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:54,228 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:54,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:54,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:54,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T21:15:54,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T21:15:54,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:54,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. as already flushing 2024-11-21T21:15:54,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:54,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:54,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:54,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:54,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:54,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44290 deadline: 1732223814390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:54,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:54,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:54,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:54,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44306 deadline: 1732223814390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:54,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:54,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:54,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:54,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:54,422 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/19c0269a9ee34b1d8341cda515c53882 2024-11-21T21:15:54,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/4ec38aad492e45b3aba8bdb5a377e1b5 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/4ec38aad492e45b3aba8bdb5a377e1b5 2024-11-21T21:15:54,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/4ec38aad492e45b3aba8bdb5a377e1b5, entries=200, sequenceid=135, filesize=38.8 K 2024-11-21T21:15:54,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/a3434d3b12c04157a1fab47eb20ffb49 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/a3434d3b12c04157a1fab47eb20ffb49 2024-11-21T21:15:54,432 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/a3434d3b12c04157a1fab47eb20ffb49, entries=150, sequenceid=135, filesize=11.9 K 2024-11-21T21:15:54,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/19c0269a9ee34b1d8341cda515c53882 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/19c0269a9ee34b1d8341cda515c53882 2024-11-21T21:15:54,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/19c0269a9ee34b1d8341cda515c53882, entries=150, sequenceid=135, filesize=11.9 K 2024-11-21T21:15:54,436 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for ca977aeb13804530742b16bc2c00bf08 in 1270ms, sequenceid=135, compaction requested=true 2024-11-21T21:15:54,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca977aeb13804530742b16bc2c00bf08: 2024-11-21T21:15:54,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca977aeb13804530742b16bc2c00bf08:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:15:54,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:54,436 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:15:54,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca977aeb13804530742b16bc2c00bf08:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:15:54,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:54,436 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:15:54,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca977aeb13804530742b16bc2c00bf08:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:15:54,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:15:54,437 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:15:54,437 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): ca977aeb13804530742b16bc2c00bf08/A is initiating minor compaction (all files) 2024-11-21T21:15:54,437 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca977aeb13804530742b16bc2c00bf08/A in TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:54,437 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/25b44c4d01744bbc8dbc9eadf9840fb4, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/b7f0096fc5aa4c01b6d2cc36de776428, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/4ec38aad492e45b3aba8bdb5a377e1b5] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp, totalSize=99.5 K 2024-11-21T21:15:54,437 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:54,437 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. files: [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/25b44c4d01744bbc8dbc9eadf9840fb4, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/b7f0096fc5aa4c01b6d2cc36de776428, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/4ec38aad492e45b3aba8bdb5a377e1b5] 2024-11-21T21:15:54,438 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:15:54,438 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): ca977aeb13804530742b16bc2c00bf08/B is initiating minor compaction (all files) 2024-11-21T21:15:54,438 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25b44c4d01744bbc8dbc9eadf9840fb4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732223750069 2024-11-21T21:15:54,438 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca977aeb13804530742b16bc2c00bf08/B in TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:54,438 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/74efbfadb8e745129f8539369ba18b1f, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/34e6f147e7504862a749912eb7d95e2e, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/a3434d3b12c04157a1fab47eb20ffb49] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp, totalSize=35.5 K 2024-11-21T21:15:54,438 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7f0096fc5aa4c01b6d2cc36de776428, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1732223751281 2024-11-21T21:15:54,438 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 74efbfadb8e745129f8539369ba18b1f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732223750069 2024-11-21T21:15:54,438 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ec38aad492e45b3aba8bdb5a377e1b5, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732223751993 2024-11-21T21:15:54,438 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 34e6f147e7504862a749912eb7d95e2e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1732223751281 2024-11-21T21:15:54,438 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting a3434d3b12c04157a1fab47eb20ffb49, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732223751993 2024-11-21T21:15:54,443 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ca977aeb13804530742b16bc2c00bf08] 2024-11-21T21:15:54,444 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411215f70cee88f27469fa544a5900915519d_ca977aeb13804530742b16bc2c00bf08 store=[table=TestAcidGuarantees family=A region=ca977aeb13804530742b16bc2c00bf08] 2024-11-21T21:15:54,445 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca977aeb13804530742b16bc2c00bf08#B#compaction#342 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:54,445 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/ff681f917f8d43f890a5314ad32ce222 is 50, key is test_row_0/B:col10/1732223753165/Put/seqid=0 2024-11-21T21:15:54,446 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411215f70cee88f27469fa544a5900915519d_ca977aeb13804530742b16bc2c00bf08, store=[table=TestAcidGuarantees family=A region=ca977aeb13804530742b16bc2c00bf08] 2024-11-21T21:15:54,446 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411215f70cee88f27469fa544a5900915519d_ca977aeb13804530742b16bc2c00bf08 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ca977aeb13804530742b16bc2c00bf08] 2024-11-21T21:15:54,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742216_1392 (size=12459) 2024-11-21T21:15:54,454 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/ff681f917f8d43f890a5314ad32ce222 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/ff681f917f8d43f890a5314ad32ce222 2024-11-21T21:15:54,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742217_1393 (size=4469) 2024-11-21T21:15:54,458 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca977aeb13804530742b16bc2c00bf08/B of ca977aeb13804530742b16bc2c00bf08 into ff681f917f8d43f890a5314ad32ce222(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:54,458 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca977aeb13804530742b16bc2c00bf08: 2024-11-21T21:15:54,458 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., storeName=ca977aeb13804530742b16bc2c00bf08/B, priority=13, startTime=1732223754436; duration=0sec 2024-11-21T21:15:54,458 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:15:54,458 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca977aeb13804530742b16bc2c00bf08:B 2024-11-21T21:15:54,458 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:15:54,459 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:15:54,459 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): ca977aeb13804530742b16bc2c00bf08/C is initiating minor compaction (all files) 2024-11-21T21:15:54,459 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca977aeb13804530742b16bc2c00bf08/C in TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:54,459 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/1d48266e64874a9d999f5ae05785c765, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/6ad36bbe4741445d974a1733f083c8a9, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/19c0269a9ee34b1d8341cda515c53882] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp, totalSize=35.5 K 2024-11-21T21:15:54,459 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d48266e64874a9d999f5ae05785c765, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732223750069 2024-11-21T21:15:54,460 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ad36bbe4741445d974a1733f083c8a9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1732223751281 2024-11-21T21:15:54,460 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 19c0269a9ee34b1d8341cda515c53882, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732223751993 2024-11-21T21:15:54,464 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca977aeb13804530742b16bc2c00bf08#C#compaction#343 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:54,465 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/9432d51697c641f686fe0317d66c3529 is 50, key is test_row_0/C:col10/1732223753165/Put/seqid=0 2024-11-21T21:15:54,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742218_1394 (size=12459) 2024-11-21T21:15:54,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:54,495 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca977aeb13804530742b16bc2c00bf08 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-21T21:15:54,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=A 2024-11-21T21:15:54,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:54,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=B 2024-11-21T21:15:54,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:54,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=C 2024-11-21T21:15:54,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:54,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121de83300f12004d9cbe06d418ce29f8a2_ca977aeb13804530742b16bc2c00bf08 is 50, key is test_row_0/A:col10/1732223753234/Put/seqid=0 2024-11-21T21:15:54,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742219_1395 (size=14794) 2024-11-21T21:15:54,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T21:15:54,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:54,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. as already flushing 2024-11-21T21:15:54,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:54,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:54,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:54,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:54,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:54,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44320 deadline: 1732223814528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:54,538 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:54,538 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:54,538 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:54,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:54,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44292 deadline: 1732223814537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:54,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:54,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:54,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:54,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:54,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44318 deadline: 1732223814538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:54,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:54,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:54,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:54,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T21:15:54,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:54,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44320 deadline: 1732223814640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:54,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:54,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:54,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:54,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:54,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44292 deadline: 1732223814650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:54,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:54,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:54,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44318 deadline: 1732223814650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:54,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:54,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:54,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:54,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:54,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:54,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T21:15:54,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:54,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. as already flushing 2024-11-21T21:15:54,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:54,687 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:54,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:54,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:54,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T21:15:54,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:54,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. as already flushing 2024-11-21T21:15:54,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:54,839 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:54,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:54,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:54,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:54,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44320 deadline: 1732223814850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:54,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:54,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:54,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:54,858 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca977aeb13804530742b16bc2c00bf08#A#compaction#341 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:54,859 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/35dd235b6df24f0aa50268b9f63faf94 is 175, key is test_row_0/A:col10/1732223753165/Put/seqid=0 2024-11-21T21:15:54,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742220_1396 (size=31413) 2024-11-21T21:15:54,873 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/9432d51697c641f686fe0317d66c3529 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/9432d51697c641f686fe0317d66c3529 2024-11-21T21:15:54,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:54,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44292 deadline: 1732223814860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:54,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:54,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44318 deadline: 1732223814860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:54,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:54,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:54,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:54,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:54,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:54,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:54,877 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca977aeb13804530742b16bc2c00bf08/C of ca977aeb13804530742b16bc2c00bf08 into 9432d51697c641f686fe0317d66c3529(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:54,877 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca977aeb13804530742b16bc2c00bf08: 2024-11-21T21:15:54,877 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., storeName=ca977aeb13804530742b16bc2c00bf08/C, priority=13, startTime=1732223754436; duration=0sec 2024-11-21T21:15:54,877 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:54,877 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca977aeb13804530742b16bc2c00bf08:C 2024-11-21T21:15:54,908 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:15:54,911 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121de83300f12004d9cbe06d418ce29f8a2_ca977aeb13804530742b16bc2c00bf08 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121de83300f12004d9cbe06d418ce29f8a2_ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:54,911 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/cef4d45b212843e18314d7cbba19ffe6, store: [table=TestAcidGuarantees family=A region=ca977aeb13804530742b16bc2c00bf08] 2024-11-21T21:15:54,912 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/cef4d45b212843e18314d7cbba19ffe6 is 175, key is test_row_0/A:col10/1732223753234/Put/seqid=0 2024-11-21T21:15:54,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742221_1397 (size=39749) 2024-11-21T21:15:54,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T21:15:54,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:54,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. as already flushing 2024-11-21T21:15:54,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:54,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:54,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:54,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:55,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T21:15:55,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T21:15:55,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:55,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. as already flushing 2024-11-21T21:15:55,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:55,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:55,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:55,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:55,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:55,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44320 deadline: 1732223815160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:55,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:55,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:55,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:55,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:55,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44292 deadline: 1732223815181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:55,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:55,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:55,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44318 deadline: 1732223815181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:55,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:55,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:55,187 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:55,187 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:55,187 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:55,267 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/35dd235b6df24f0aa50268b9f63faf94 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/35dd235b6df24f0aa50268b9f63faf94 2024-11-21T21:15:55,271 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca977aeb13804530742b16bc2c00bf08/A of ca977aeb13804530742b16bc2c00bf08 into 35dd235b6df24f0aa50268b9f63faf94(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:55,271 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca977aeb13804530742b16bc2c00bf08: 2024-11-21T21:15:55,271 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., storeName=ca977aeb13804530742b16bc2c00bf08/A, priority=13, startTime=1732223754436; duration=0sec 2024-11-21T21:15:55,271 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:55,271 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca977aeb13804530742b16bc2c00bf08:A 2024-11-21T21:15:55,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T21:15:55,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:55,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. as already flushing 2024-11-21T21:15:55,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:55,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:55,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:55,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:55,315 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/cef4d45b212843e18314d7cbba19ffe6 2024-11-21T21:15:55,320 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/65d90193b6a1417ba9f74cd568aca39b is 50, key is test_row_0/B:col10/1732223753234/Put/seqid=0 2024-11-21T21:15:55,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742222_1398 (size=12151) 2024-11-21T21:15:55,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:55,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:55,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44306 deadline: 1732223815411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:55,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44290 deadline: 1732223815411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:55,419 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,41213,1732223653469 for 'test_row_1' in ca977aeb13804530742b16bc2c00bf08 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:15:55,419 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,41213,1732223653469 for 'test_row_1' in ca977aeb13804530742b16bc2c00bf08 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:15:55,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:55,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:55,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:55,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:55,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:55,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:55,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T21:15:55,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:55,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. as already flushing 2024-11-21T21:15:55,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:55,450 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:55,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:55,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:55,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T21:15:55,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:55,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. as already flushing 2024-11-21T21:15:55,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:55,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:55,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:55,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:55,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:55,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44320 deadline: 1732223815670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:55,678 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:55,678 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:55,678 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:55,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:55,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44292 deadline: 1732223815700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:55,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:55,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44318 deadline: 1732223815700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:55,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:55,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:55,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:55,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:55,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:55,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:55,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/65d90193b6a1417ba9f74cd568aca39b 2024-11-21T21:15:55,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/0350ed38bbf8485f8584a6e831feb848 is 50, key is test_row_0/C:col10/1732223753234/Put/seqid=0 2024-11-21T21:15:55,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742223_1399 (size=12151) 2024-11-21T21:15:55,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T21:15:55,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:55,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. as already flushing 2024-11-21T21:15:55,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:55,755 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:55,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:55,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:55,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T21:15:55,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:55,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. as already flushing 2024-11-21T21:15:55,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:55,908 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:55,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:55,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:56,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T21:15:56,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:56,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. as already flushing 2024-11-21T21:15:56,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:56,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:56,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:56,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:56,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T21:15:56,142 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/0350ed38bbf8485f8584a6e831feb848 2024-11-21T21:15:56,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/cef4d45b212843e18314d7cbba19ffe6 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/cef4d45b212843e18314d7cbba19ffe6 2024-11-21T21:15:56,149 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/cef4d45b212843e18314d7cbba19ffe6, entries=200, sequenceid=155, filesize=38.8 K 2024-11-21T21:15:56,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/65d90193b6a1417ba9f74cd568aca39b as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/65d90193b6a1417ba9f74cd568aca39b 2024-11-21T21:15:56,152 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/65d90193b6a1417ba9f74cd568aca39b, entries=150, sequenceid=155, filesize=11.9 K 2024-11-21T21:15:56,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/0350ed38bbf8485f8584a6e831feb848 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/0350ed38bbf8485f8584a6e831feb848 2024-11-21T21:15:56,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/0350ed38bbf8485f8584a6e831feb848, entries=150, sequenceid=155, filesize=11.9 K 2024-11-21T21:15:56,156 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for ca977aeb13804530742b16bc2c00bf08 in 1661ms, sequenceid=155, compaction requested=false 2024-11-21T21:15:56,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca977aeb13804530742b16bc2c00bf08: 2024-11-21T21:15:56,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T21:15:56,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:56,213 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing ca977aeb13804530742b16bc2c00bf08 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-21T21:15:56,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=A 2024-11-21T21:15:56,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:56,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=B 2024-11-21T21:15:56,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:56,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=C 2024-11-21T21:15:56,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:56,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d27074ca01be427aa3db1d1152daa688_ca977aeb13804530742b16bc2c00bf08 is 50, key is test_row_0/A:col10/1732223754537/Put/seqid=0 2024-11-21T21:15:56,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742224_1400 (size=12304) 2024-11-21T21:15:56,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:15:56,630 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d27074ca01be427aa3db1d1152daa688_ca977aeb13804530742b16bc2c00bf08 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d27074ca01be427aa3db1d1152daa688_ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:56,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/c3b924adfce54121bc6509fc1f8ba491, store: [table=TestAcidGuarantees family=A region=ca977aeb13804530742b16bc2c00bf08] 2024-11-21T21:15:56,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/c3b924adfce54121bc6509fc1f8ba491 is 175, key is test_row_0/A:col10/1732223754537/Put/seqid=0 2024-11-21T21:15:56,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742225_1401 (size=31105) 2024-11-21T21:15:56,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:56,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. as already flushing 2024-11-21T21:15:56,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:56,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44318 deadline: 1732223816785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:56,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:56,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:56,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:56,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:56,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44292 deadline: 1732223816787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:56,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:56,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44320 deadline: 1732223816787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:56,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:56,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:56,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:56,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:56,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:56,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:56,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:56,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44318 deadline: 1732223816900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:56,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:56,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:56,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:56,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:56,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:56,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44292 deadline: 1732223816910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:56,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44320 deadline: 1732223816910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:56,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:56,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:56,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:56,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:56,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:56,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:57,035 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/c3b924adfce54121bc6509fc1f8ba491 2024-11-21T21:15:57,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/fc1e5373558f48afa0aaccdee154cf94 is 50, key is test_row_0/B:col10/1732223754537/Put/seqid=0 2024-11-21T21:15:57,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742226_1402 (size=12151) 2024-11-21T21:15:57,044 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/fc1e5373558f48afa0aaccdee154cf94 2024-11-21T21:15:57,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/68a10e9963de45d6b6a054dc964aeba3 is 50, key is test_row_0/C:col10/1732223754537/Put/seqid=0 2024-11-21T21:15:57,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742227_1403 (size=12151) 2024-11-21T21:15:57,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:57,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44318 deadline: 1732223817120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:57,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:57,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:57,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:57,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44292 deadline: 1732223817130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:57,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:57,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:57,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:57,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44320 deadline: 1732223817130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:57,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:57,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:57,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:57,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:57,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44306 deadline: 1732223817430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:57,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:57,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44318 deadline: 1732223817430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:57,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:57,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:57,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:57,432 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,41213,1732223653469 for 'test_row_1' in ca977aeb13804530742b16bc2c00bf08 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:15:57,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:57,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:57,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:57,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:57,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44290 deadline: 1732223817440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:57,445 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,41213,1732223653469 for 'test_row_1' in ca977aeb13804530742b16bc2c00bf08 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:15:57,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:57,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:57,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:57,456 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/68a10e9963de45d6b6a054dc964aeba3 2024-11-21T21:15:57,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:57,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44292 deadline: 1732223817451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:57,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:57,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:57,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44320 deadline: 1732223817451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:57,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:57,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:57,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:57,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:57,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:57,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/c3b924adfce54121bc6509fc1f8ba491 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/c3b924adfce54121bc6509fc1f8ba491 2024-11-21T21:15:57,465 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/c3b924adfce54121bc6509fc1f8ba491, entries=150, sequenceid=170, filesize=30.4 K 2024-11-21T21:15:57,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/fc1e5373558f48afa0aaccdee154cf94 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/fc1e5373558f48afa0aaccdee154cf94 2024-11-21T21:15:57,469 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/fc1e5373558f48afa0aaccdee154cf94, entries=150, sequenceid=170, filesize=11.9 K 2024-11-21T21:15:57,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/68a10e9963de45d6b6a054dc964aeba3 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/68a10e9963de45d6b6a054dc964aeba3 2024-11-21T21:15:57,473 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/68a10e9963de45d6b6a054dc964aeba3, entries=150, sequenceid=170, filesize=11.9 K 2024-11-21T21:15:57,474 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for ca977aeb13804530742b16bc2c00bf08 in 1261ms, sequenceid=170, compaction requested=true 2024-11-21T21:15:57,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for ca977aeb13804530742b16bc2c00bf08: 2024-11-21T21:15:57,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:57,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-21T21:15:57,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-21T21:15:57,478 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-21T21:15:57,478 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5530 sec 2024-11-21T21:15:57,485 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 3.5580 sec 2024-11-21T21:15:57,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:57,943 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca977aeb13804530742b16bc2c00bf08 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-21T21:15:57,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=A 2024-11-21T21:15:57,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:15:57,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=B 2024-11-21T21:15:57,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:15:57,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=C 2024-11-21T21:15:57,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:15:57,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411214451f375fbf94cc0a827b8b24a8cdbc1_ca977aeb13804530742b16bc2c00bf08 is 50, key is test_row_0/A:col10/1732223756784/Put/seqid=0 2024-11-21T21:15:57,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742228_1404 (size=14794) 2024-11-21T21:15:57,953 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:15:57,955 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411214451f375fbf94cc0a827b8b24a8cdbc1_ca977aeb13804530742b16bc2c00bf08 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214451f375fbf94cc0a827b8b24a8cdbc1_ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:57,956 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/4915a280f99f4f158c8f7041955af399, store: [table=TestAcidGuarantees family=A region=ca977aeb13804530742b16bc2c00bf08] 2024-11-21T21:15:57,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/4915a280f99f4f158c8f7041955af399 is 175, key is test_row_0/A:col10/1732223756784/Put/seqid=0 2024-11-21T21:15:57,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742229_1405 (size=39749) 2024-11-21T21:15:57,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:57,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44318 deadline: 1732223817979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:57,985 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:57,985 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:57,985 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:57,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:57,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44292 deadline: 1732223817984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:57,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:57,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:57,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:57,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44320 deadline: 1732223817985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:57,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:57,998 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:57,998 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:57,998 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:58,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T21:15:58,090 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:15:58,091 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:15:58,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:58,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44318 deadline: 1732223818090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:58,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:58,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:58,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:58,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-21T21:15:58,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-21T21:15:58,092 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:15:58,093 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:15:58,093 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:15:58,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:58,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44292 deadline: 1732223818100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:58,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:58,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:58,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:58,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:58,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44320 deadline: 1732223818100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:58,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:58,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:58,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:58,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-21T21:15:58,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-21T21:15:58,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:58,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. as already flushing 2024-11-21T21:15:58,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:58,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:58,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:58,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:58,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:58,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44318 deadline: 1732223818300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:58,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:58,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:58,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:58,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:58,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44292 deadline: 1732223818310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:58,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:58,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44320 deadline: 1732223818311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:58,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:58,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:58,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:58,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:58,317 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:58,317 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:58,360 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=191, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/4915a280f99f4f158c8f7041955af399 2024-11-21T21:15:58,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/292faf3c18be4d94a0f833de1b1887e9 is 50, key is test_row_0/B:col10/1732223756784/Put/seqid=0 2024-11-21T21:15:58,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742230_1406 (size=12151) 2024-11-21T21:15:58,374 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/292faf3c18be4d94a0f833de1b1887e9 2024-11-21T21:15:58,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/0b5ad926b8c942aa8b5ed4c55cc3df9a is 50, key is test_row_0/C:col10/1732223756784/Put/seqid=0 2024-11-21T21:15:58,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742231_1407 (size=12151) 2024-11-21T21:15:58,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-21T21:15:58,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:58,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. as already flushing 2024-11-21T21:15:58,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:58,398 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:58,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:58,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:58,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-21T21:15:58,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-21T21:15:58,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:58,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. as already flushing 2024-11-21T21:15:58,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:58,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:58,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:58,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:58,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:58,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44318 deadline: 1732223818610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:58,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:58,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:58,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:58,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:58,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44292 deadline: 1732223818620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:58,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:58,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:58,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:58,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:58,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44320 deadline: 1732223818620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:58,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:58,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:58,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:58,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-21T21:15:58,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:58,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. as already flushing 2024-11-21T21:15:58,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:58,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:58,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:58,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:15:58,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-21T21:15:58,790 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/0b5ad926b8c942aa8b5ed4c55cc3df9a 2024-11-21T21:15:58,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/4915a280f99f4f158c8f7041955af399 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/4915a280f99f4f158c8f7041955af399 2024-11-21T21:15:58,809 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/4915a280f99f4f158c8f7041955af399, entries=200, sequenceid=191, filesize=38.8 K 2024-11-21T21:15:58,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/292faf3c18be4d94a0f833de1b1887e9 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/292faf3c18be4d94a0f833de1b1887e9 2024-11-21T21:15:58,816 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/292faf3c18be4d94a0f833de1b1887e9, entries=150, sequenceid=191, filesize=11.9 K 2024-11-21T21:15:58,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/0b5ad926b8c942aa8b5ed4c55cc3df9a as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/0b5ad926b8c942aa8b5ed4c55cc3df9a 2024-11-21T21:15:58,822 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/0b5ad926b8c942aa8b5ed4c55cc3df9a, entries=150, sequenceid=191, filesize=11.9 K 2024-11-21T21:15:58,823 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for ca977aeb13804530742b16bc2c00bf08 in 880ms, sequenceid=191, compaction requested=true 2024-11-21T21:15:58,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca977aeb13804530742b16bc2c00bf08: 2024-11-21T21:15:58,823 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:15:58,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca977aeb13804530742b16bc2c00bf08:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:15:58,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:58,823 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:15:58,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca977aeb13804530742b16bc2c00bf08:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:15:58,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:58,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca977aeb13804530742b16bc2c00bf08:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:15:58,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:15:58,827 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142016 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:15:58,827 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): ca977aeb13804530742b16bc2c00bf08/A is initiating minor compaction (all files) 2024-11-21T21:15:58,827 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca977aeb13804530742b16bc2c00bf08/A in TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:58,827 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:15:58,827 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): ca977aeb13804530742b16bc2c00bf08/B is initiating minor compaction (all files) 2024-11-21T21:15:58,827 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/35dd235b6df24f0aa50268b9f63faf94, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/cef4d45b212843e18314d7cbba19ffe6, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/c3b924adfce54121bc6509fc1f8ba491, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/4915a280f99f4f158c8f7041955af399] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp, totalSize=138.7 K 2024-11-21T21:15:58,827 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca977aeb13804530742b16bc2c00bf08/B in TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:58,827 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:58,827 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/ff681f917f8d43f890a5314ad32ce222, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/65d90193b6a1417ba9f74cd568aca39b, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/fc1e5373558f48afa0aaccdee154cf94, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/292faf3c18be4d94a0f833de1b1887e9] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp, totalSize=47.8 K 2024-11-21T21:15:58,827 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. files: [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/35dd235b6df24f0aa50268b9f63faf94, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/cef4d45b212843e18314d7cbba19ffe6, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/c3b924adfce54121bc6509fc1f8ba491, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/4915a280f99f4f158c8f7041955af399] 2024-11-21T21:15:58,827 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting ff681f917f8d43f890a5314ad32ce222, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732223751993 2024-11-21T21:15:58,827 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35dd235b6df24f0aa50268b9f63faf94, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732223751993 2024-11-21T21:15:58,828 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 65d90193b6a1417ba9f74cd568aca39b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732223753220 2024-11-21T21:15:58,828 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting cef4d45b212843e18314d7cbba19ffe6, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732223753220 2024-11-21T21:15:58,828 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting fc1e5373558f48afa0aaccdee154cf94, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732223754521 2024-11-21T21:15:58,828 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3b924adfce54121bc6509fc1f8ba491, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732223754521 2024-11-21T21:15:58,828 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 292faf3c18be4d94a0f833de1b1887e9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732223756784 2024-11-21T21:15:58,828 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4915a280f99f4f158c8f7041955af399, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732223756747 2024-11-21T21:15:58,839 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ca977aeb13804530742b16bc2c00bf08] 2024-11-21T21:15:58,847 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca977aeb13804530742b16bc2c00bf08#B#compaction#354 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:58,848 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/ffc706e19a38481aa891eeb04b22fb70 is 50, key is test_row_0/B:col10/1732223756784/Put/seqid=0 2024-11-21T21:15:58,849 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121ddf63fcdcfe04d37b69cb6b4f99d9a04_ca977aeb13804530742b16bc2c00bf08 store=[table=TestAcidGuarantees family=A region=ca977aeb13804530742b16bc2c00bf08] 2024-11-21T21:15:58,852 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121ddf63fcdcfe04d37b69cb6b4f99d9a04_ca977aeb13804530742b16bc2c00bf08, store=[table=TestAcidGuarantees family=A region=ca977aeb13804530742b16bc2c00bf08] 2024-11-21T21:15:58,852 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ddf63fcdcfe04d37b69cb6b4f99d9a04_ca977aeb13804530742b16bc2c00bf08 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ca977aeb13804530742b16bc2c00bf08] 2024-11-21T21:15:58,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742232_1408 (size=12595) 2024-11-21T21:15:58,864 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/ffc706e19a38481aa891eeb04b22fb70 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/ffc706e19a38481aa891eeb04b22fb70 2024-11-21T21:15:58,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-21T21:15:58,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:58,864 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing ca977aeb13804530742b16bc2c00bf08 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-21T21:15:58,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=A 2024-11-21T21:15:58,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:58,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=B 2024-11-21T21:15:58,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:58,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=C 2024-11-21T21:15:58,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:15:58,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742233_1409 (size=4469) 2024-11-21T21:15:58,871 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ca977aeb13804530742b16bc2c00bf08/B of ca977aeb13804530742b16bc2c00bf08 into ffc706e19a38481aa891eeb04b22fb70(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:58,871 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca977aeb13804530742b16bc2c00bf08: 2024-11-21T21:15:58,871 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., storeName=ca977aeb13804530742b16bc2c00bf08/B, priority=12, startTime=1732223758823; duration=0sec 2024-11-21T21:15:58,871 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:15:58,871 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca977aeb13804530742b16bc2c00bf08:B 2024-11-21T21:15:58,871 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:15:58,879 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:15:58,879 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): ca977aeb13804530742b16bc2c00bf08/C is initiating minor compaction (all files) 2024-11-21T21:15:58,879 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca977aeb13804530742b16bc2c00bf08/C in TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:58,879 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/9432d51697c641f686fe0317d66c3529, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/0350ed38bbf8485f8584a6e831feb848, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/68a10e9963de45d6b6a054dc964aeba3, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/0b5ad926b8c942aa8b5ed4c55cc3df9a] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp, totalSize=47.8 K 2024-11-21T21:15:58,880 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 9432d51697c641f686fe0317d66c3529, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732223751993 2024-11-21T21:15:58,880 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 0350ed38bbf8485f8584a6e831feb848, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732223753220 2024-11-21T21:15:58,880 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 68a10e9963de45d6b6a054dc964aeba3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732223754521 2024-11-21T21:15:58,881 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b5ad926b8c942aa8b5ed4c55cc3df9a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732223756784 2024-11-21T21:15:58,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112161b97f4457844baeae9a773be282d923_ca977aeb13804530742b16bc2c00bf08 is 50, key is test_row_0/A:col10/1732223757978/Put/seqid=0 2024-11-21T21:15:58,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742234_1410 (size=12304) 2024-11-21T21:15:58,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:15:58,904 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca977aeb13804530742b16bc2c00bf08#C#compaction#356 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:58,904 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/49aa739294644390bd7bab1a45d50658 is 50, key is test_row_0/C:col10/1732223756784/Put/seqid=0 2024-11-21T21:15:58,910 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112161b97f4457844baeae9a773be282d923_ca977aeb13804530742b16bc2c00bf08 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112161b97f4457844baeae9a773be282d923_ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:58,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742235_1411 (size=12595) 2024-11-21T21:15:58,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/51c23b6fec194c2da993853c10dcc0fb, store: [table=TestAcidGuarantees family=A region=ca977aeb13804530742b16bc2c00bf08] 2024-11-21T21:15:58,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/51c23b6fec194c2da993853c10dcc0fb is 175, key is test_row_0/A:col10/1732223757978/Put/seqid=0 2024-11-21T21:15:58,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742236_1412 (size=31105) 2024-11-21T21:15:59,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:59,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. as already flushing 2024-11-21T21:15:59,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:59,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44320 deadline: 1732223819199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:59,209 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:59,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732223759184, max timestamp=1732223759206Num uniques 859; 2024-11-21T21:15:59,209 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:59,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732223759184, max timestamp=1732223759206Num uniques 859; 2024-11-21T21:15:59,209 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:59,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T21:15:59,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T21:15:59,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732223759184, max timestamp=1732223759206Num uniques 859; 2024-11-21T21:15:59,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T21:15:59,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-21T21:15:59,267 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca977aeb13804530742b16bc2c00bf08#A#compaction#353 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:15:59,267 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/f9a7e81403e646a8adde08553be7e1f4 is 175, key is test_row_0/A:col10/1732223756784/Put/seqid=0 2024-11-21T21:15:59,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742237_1413 (size=31549) 2024-11-21T21:15:59,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:59,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44318 deadline: 1732223819272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:59,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:59,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:59,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:59,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:59,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44292 deadline: 1732223819274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:59,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:59,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:59,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:59,315 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/49aa739294644390bd7bab1a45d50658 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/49aa739294644390bd7bab1a45d50658 2024-11-21T21:15:59,318 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=202, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/51c23b6fec194c2da993853c10dcc0fb 2024-11-21T21:15:59,319 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ca977aeb13804530742b16bc2c00bf08/C of ca977aeb13804530742b16bc2c00bf08 into 49aa739294644390bd7bab1a45d50658(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:59,319 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca977aeb13804530742b16bc2c00bf08: 2024-11-21T21:15:59,319 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., storeName=ca977aeb13804530742b16bc2c00bf08/C, priority=12, startTime=1732223758823; duration=0sec 2024-11-21T21:15:59,319 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:59,319 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca977aeb13804530742b16bc2c00bf08:C 2024-11-21T21:15:59,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/bd53eab6b16744b482be2c1c0e513902 is 50, key is test_row_0/B:col10/1732223757978/Put/seqid=0 2024-11-21T21:15:59,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742238_1414 (size=12151) 2024-11-21T21:15:59,329 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/bd53eab6b16744b482be2c1c0e513902 2024-11-21T21:15:59,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:59,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44320 deadline: 1732223819320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:59,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:59,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:59,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:59,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/303ca5d0b24a481183e483180aa6d3c1 is 50, key is test_row_0/C:col10/1732223757978/Put/seqid=0 2024-11-21T21:15:59,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742239_1415 (size=12151) 2024-11-21T21:15:59,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:59,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44318 deadline: 1732223819390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:59,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:59,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:59,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:59,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:59,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44292 deadline: 1732223819400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:59,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:59,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:59,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:59,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:59,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44320 deadline: 1732223819540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:59,550 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:59,550 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:59,550 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:59,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:59,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44318 deadline: 1732223819600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:59,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:59,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:59,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:59,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:59,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44292 deadline: 1732223819620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:59,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:59,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:59,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:59,677 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/f9a7e81403e646a8adde08553be7e1f4 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/f9a7e81403e646a8adde08553be7e1f4 2024-11-21T21:15:59,681 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ca977aeb13804530742b16bc2c00bf08/A of ca977aeb13804530742b16bc2c00bf08 into f9a7e81403e646a8adde08553be7e1f4(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:15:59,681 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca977aeb13804530742b16bc2c00bf08: 2024-11-21T21:15:59,681 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., storeName=ca977aeb13804530742b16bc2c00bf08/A, priority=12, startTime=1732223758823; duration=0sec 2024-11-21T21:15:59,681 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:15:59,681 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca977aeb13804530742b16bc2c00bf08:A 2024-11-21T21:15:59,744 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/303ca5d0b24a481183e483180aa6d3c1 2024-11-21T21:15:59,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/51c23b6fec194c2da993853c10dcc0fb as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/51c23b6fec194c2da993853c10dcc0fb 2024-11-21T21:15:59,751 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/51c23b6fec194c2da993853c10dcc0fb, entries=150, sequenceid=202, filesize=30.4 K 2024-11-21T21:15:59,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/bd53eab6b16744b482be2c1c0e513902 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/bd53eab6b16744b482be2c1c0e513902 2024-11-21T21:15:59,755 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/bd53eab6b16744b482be2c1c0e513902, entries=150, sequenceid=202, filesize=11.9 K 2024-11-21T21:15:59,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/303ca5d0b24a481183e483180aa6d3c1 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/303ca5d0b24a481183e483180aa6d3c1 2024-11-21T21:15:59,759 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/303ca5d0b24a481183e483180aa6d3c1, entries=150, sequenceid=202, filesize=11.9 K 2024-11-21T21:15:59,759 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for ca977aeb13804530742b16bc2c00bf08 in 895ms, sequenceid=202, compaction requested=false 2024-11-21T21:15:59,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for ca977aeb13804530742b16bc2c00bf08: 2024-11-21T21:15:59,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:15:59,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-21T21:15:59,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-21T21:15:59,762 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-21T21:15:59,762 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6680 sec 2024-11-21T21:15:59,764 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.6720 sec 2024-11-21T21:15:59,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:15:59,868 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca977aeb13804530742b16bc2c00bf08 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-21T21:15:59,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=A 2024-11-21T21:15:59,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:15:59,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=B 2024-11-21T21:15:59,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:15:59,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=C 2024-11-21T21:15:59,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:15:59,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411215906f1ca878449789ad656deafc7d4a2_ca977aeb13804530742b16bc2c00bf08 is 50, key is test_row_0/A:col10/1732223759242/Put/seqid=0 2024-11-21T21:15:59,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742240_1416 (size=14794) 2024-11-21T21:15:59,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:59,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44318 deadline: 1732223819920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:59,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:59,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44320 deadline: 1732223819925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:59,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:59,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:59,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:59,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:59,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:59,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:15:59,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:15:59,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44292 deadline: 1732223819930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:15:59,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:59,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:15:59,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:00,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:00,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44320 deadline: 1732223820041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:00,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:00,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:00,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:00,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-21T21:16:00,241 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:16:00,241 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:16:00,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-21T21:16:00,243 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:16:00,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-21T21:16:00,243 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:16:00,244 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:16:00,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:00,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44320 deadline: 1732223820261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:00,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:00,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:00,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:00,286 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:16:00,289 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411215906f1ca878449789ad656deafc7d4a2_ca977aeb13804530742b16bc2c00bf08 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215906f1ca878449789ad656deafc7d4a2_ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:16:00,290 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/67cf24bcc58c4d35934b0ddde14a3920, store: [table=TestAcidGuarantees family=A region=ca977aeb13804530742b16bc2c00bf08] 2024-11-21T21:16:00,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/67cf24bcc58c4d35934b0ddde14a3920 is 175, key is test_row_0/A:col10/1732223759242/Put/seqid=0 2024-11-21T21:16:00,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742241_1417 (size=39749) 2024-11-21T21:16:00,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-21T21:16:00,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-21T21:16:00,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:16:00,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. as already flushing 2024-11-21T21:16:00,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:16:00,396 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:00,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:00,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:00,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:00,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44318 deadline: 1732223820430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:00,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:00,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:00,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:00,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:00,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44292 deadline: 1732223820450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:00,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:00,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:00,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:00,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-21T21:16:00,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:16:00,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. as already flushing 2024-11-21T21:16:00,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:16:00,549 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:00,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:00,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:00,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-21T21:16:00,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:00,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44320 deadline: 1732223820580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:00,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:00,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:00,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:00,693 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=240, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/67cf24bcc58c4d35934b0ddde14a3920 2024-11-21T21:16:00,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/afb5d7914bf841ba9013a4296fcc8e5c is 50, key is test_row_0/B:col10/1732223759242/Put/seqid=0 2024-11-21T21:16:00,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-21T21:16:00,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:16:00,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. as already flushing 2024-11-21T21:16:00,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:16:00,702 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:00,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:00,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:00,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742242_1418 (size=12151) 2024-11-21T21:16:00,705 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/afb5d7914bf841ba9013a4296fcc8e5c 2024-11-21T21:16:00,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/d9a8980acf5741a8a15a5dcd7c56673e is 50, key is test_row_0/C:col10/1732223759242/Put/seqid=0 2024-11-21T21:16:00,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742243_1419 (size=12151) 2024-11-21T21:16:00,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-21T21:16:00,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:16:00,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. as already flushing 2024-11-21T21:16:00,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:16:00,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:00,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:00,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:00,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-21T21:16:01,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-21T21:16:01,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:16:01,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. as already flushing 2024-11-21T21:16:01,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:16:01,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:01,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:01,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:01,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:01,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44320 deadline: 1732223821101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:01,109 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:01,109 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:01,109 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:01,117 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/d9a8980acf5741a8a15a5dcd7c56673e 2024-11-21T21:16:01,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/67cf24bcc58c4d35934b0ddde14a3920 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/67cf24bcc58c4d35934b0ddde14a3920 2024-11-21T21:16:01,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/67cf24bcc58c4d35934b0ddde14a3920, entries=200, sequenceid=240, filesize=38.8 K 2024-11-21T21:16:01,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/afb5d7914bf841ba9013a4296fcc8e5c as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/afb5d7914bf841ba9013a4296fcc8e5c 2024-11-21T21:16:01,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/afb5d7914bf841ba9013a4296fcc8e5c, entries=150, sequenceid=240, filesize=11.9 K 2024-11-21T21:16:01,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/d9a8980acf5741a8a15a5dcd7c56673e as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/d9a8980acf5741a8a15a5dcd7c56673e 2024-11-21T21:16:01,130 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/d9a8980acf5741a8a15a5dcd7c56673e, entries=150, sequenceid=240, filesize=11.9 K 2024-11-21T21:16:01,131 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for ca977aeb13804530742b16bc2c00bf08 in 1262ms, sequenceid=240, compaction requested=true 2024-11-21T21:16:01,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca977aeb13804530742b16bc2c00bf08: 2024-11-21T21:16:01,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca977aeb13804530742b16bc2c00bf08:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:16:01,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:01,131 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:16:01,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca977aeb13804530742b16bc2c00bf08:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:16:01,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:01,131 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:16:01,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca977aeb13804530742b16bc2c00bf08:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:16:01,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:16:01,132 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:16:01,132 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:16:01,132 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): ca977aeb13804530742b16bc2c00bf08/A is initiating minor compaction (all files) 2024-11-21T21:16:01,132 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): ca977aeb13804530742b16bc2c00bf08/B is initiating minor compaction (all files) 2024-11-21T21:16:01,132 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca977aeb13804530742b16bc2c00bf08/A in TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:16:01,132 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca977aeb13804530742b16bc2c00bf08/B in TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:16:01,132 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/f9a7e81403e646a8adde08553be7e1f4, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/51c23b6fec194c2da993853c10dcc0fb, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/67cf24bcc58c4d35934b0ddde14a3920] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp, totalSize=100.0 K 2024-11-21T21:16:01,132 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/ffc706e19a38481aa891eeb04b22fb70, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/bd53eab6b16744b482be2c1c0e513902, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/afb5d7914bf841ba9013a4296fcc8e5c] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp, totalSize=36.0 K 2024-11-21T21:16:01,132 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:16:01,132 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. files: [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/f9a7e81403e646a8adde08553be7e1f4, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/51c23b6fec194c2da993853c10dcc0fb, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/67cf24bcc58c4d35934b0ddde14a3920] 2024-11-21T21:16:01,132 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting ffc706e19a38481aa891eeb04b22fb70, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732223756784 2024-11-21T21:16:01,132 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9a7e81403e646a8adde08553be7e1f4, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732223756784 2024-11-21T21:16:01,132 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting bd53eab6b16744b482be2c1c0e513902, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732223757977 2024-11-21T21:16:01,132 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51c23b6fec194c2da993853c10dcc0fb, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732223757977 2024-11-21T21:16:01,132 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting afb5d7914bf841ba9013a4296fcc8e5c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732223759242 2024-11-21T21:16:01,133 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67cf24bcc58c4d35934b0ddde14a3920, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732223759242 2024-11-21T21:16:01,137 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ca977aeb13804530742b16bc2c00bf08] 2024-11-21T21:16:01,138 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca977aeb13804530742b16bc2c00bf08#B#compaction#362 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:16:01,139 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/254fa82579a44918bff8fb96f1b72ff5 is 50, key is test_row_0/B:col10/1732223759242/Put/seqid=0 2024-11-21T21:16:01,141 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112195c17b659e24488cabfec9d1b68dafa9_ca977aeb13804530742b16bc2c00bf08 store=[table=TestAcidGuarantees family=A region=ca977aeb13804530742b16bc2c00bf08] 2024-11-21T21:16:01,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742244_1420 (size=12697) 2024-11-21T21:16:01,143 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112195c17b659e24488cabfec9d1b68dafa9_ca977aeb13804530742b16bc2c00bf08, store=[table=TestAcidGuarantees family=A region=ca977aeb13804530742b16bc2c00bf08] 2024-11-21T21:16:01,143 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112195c17b659e24488cabfec9d1b68dafa9_ca977aeb13804530742b16bc2c00bf08 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ca977aeb13804530742b16bc2c00bf08] 2024-11-21T21:16:01,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742245_1421 (size=4469) 2024-11-21T21:16:01,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-21T21:16:01,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:16:01,160 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing ca977aeb13804530742b16bc2c00bf08 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T21:16:01,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=A 2024-11-21T21:16:01,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:01,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=B 2024-11-21T21:16:01,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:01,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=C 2024-11-21T21:16:01,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:01,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112194f6429bc5f241028e2a7c86bb7a2e36_ca977aeb13804530742b16bc2c00bf08 is 50, key is test_row_0/A:col10/1732223759915/Put/seqid=0 2024-11-21T21:16:01,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742246_1422 (size=12304) 2024-11-21T21:16:01,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-21T21:16:01,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:16:01,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. as already flushing 2024-11-21T21:16:01,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:01,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44306 deadline: 1732223821511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:01,516 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732223761494, max timestamp=1732223761513Num uniques 859; 2024-11-21T21:16:01,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:01,516 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-21T21:16:01,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:01,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:01,516 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732223761494, max timestamp=1732223761513Num uniques 859; 2024-11-21T21:16:01,516 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732223761494, max timestamp=1732223761513Num uniques 859; 2024-11-21T21:16:01,516 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-21T21:16:01,516 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-21T21:16:01,546 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/254fa82579a44918bff8fb96f1b72ff5 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/254fa82579a44918bff8fb96f1b72ff5 2024-11-21T21:16:01,549 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca977aeb13804530742b16bc2c00bf08#A#compaction#363 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:16:01,550 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/4acab50a0e2a43b88b648a34b25893db is 175, key is test_row_0/A:col10/1732223759242/Put/seqid=0 2024-11-21T21:16:01,551 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca977aeb13804530742b16bc2c00bf08/B of ca977aeb13804530742b16bc2c00bf08 into 254fa82579a44918bff8fb96f1b72ff5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:16:01,551 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca977aeb13804530742b16bc2c00bf08: 2024-11-21T21:16:01,551 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., storeName=ca977aeb13804530742b16bc2c00bf08/B, priority=13, startTime=1732223761131; duration=0sec 2024-11-21T21:16:01,551 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:16:01,551 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca977aeb13804530742b16bc2c00bf08:B 2024-11-21T21:16:01,551 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:16:01,559 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:16:01,559 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): ca977aeb13804530742b16bc2c00bf08/C is initiating minor compaction (all files) 2024-11-21T21:16:01,559 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca977aeb13804530742b16bc2c00bf08/C in TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:16:01,560 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/49aa739294644390bd7bab1a45d50658, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/303ca5d0b24a481183e483180aa6d3c1, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/d9a8980acf5741a8a15a5dcd7c56673e] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp, totalSize=36.0 K 2024-11-21T21:16:01,560 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 49aa739294644390bd7bab1a45d50658, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732223756784 2024-11-21T21:16:01,560 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 303ca5d0b24a481183e483180aa6d3c1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732223757977 2024-11-21T21:16:01,560 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting d9a8980acf5741a8a15a5dcd7c56673e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732223759242 2024-11-21T21:16:01,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742247_1423 (size=31651) 2024-11-21T21:16:01,566 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca977aeb13804530742b16bc2c00bf08#C#compaction#365 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:16:01,566 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/5ad34e5b311542a0a7b21b5116bedf79 is 50, key is test_row_0/C:col10/1732223759242/Put/seqid=0 2024-11-21T21:16:01,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742248_1424 (size=12697) 2024-11-21T21:16:01,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:16:01,574 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112194f6429bc5f241028e2a7c86bb7a2e36_ca977aeb13804530742b16bc2c00bf08 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112194f6429bc5f241028e2a7c86bb7a2e36_ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:16:01,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/60731f36d53947ab85d357334cb6cc54, store: [table=TestAcidGuarantees family=A region=ca977aeb13804530742b16bc2c00bf08] 2024-11-21T21:16:01,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/60731f36d53947ab85d357334cb6cc54 is 175, key is test_row_0/A:col10/1732223759915/Put/seqid=0 2024-11-21T21:16:01,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742249_1425 (size=31105) 2024-11-21T21:16:01,579 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=250, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/60731f36d53947ab85d357334cb6cc54 2024-11-21T21:16:01,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:01,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44318 deadline: 1732223821568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:01,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:01,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:01,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:01,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/62f2377e4a314b9791b54c1308367e68 is 50, key is test_row_0/B:col10/1732223759915/Put/seqid=0 2024-11-21T21:16:01,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:01,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44290 deadline: 1732223821576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:01,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:01,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44292 deadline: 1732223821576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:01,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:01,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:01,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:01,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:01,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:01,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:01,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742250_1426 (size=12151) 2024-11-21T21:16:01,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:01,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44306 deadline: 1732223821620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:01,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:01,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:01,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:01,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:01,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44318 deadline: 1732223821690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:01,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:01,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:01,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:01,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:01,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44290 deadline: 1732223821690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:01,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:01,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:01,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:01,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:01,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44292 deadline: 1732223821690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:01,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:01,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:01,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:01,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:01,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44306 deadline: 1732223821840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:01,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:01,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:01,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:01,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:01,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44318 deadline: 1732223821910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:01,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:01,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:01,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:01,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44290 deadline: 1732223821910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:01,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:01,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:01,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:01,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:01,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:01,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44292 deadline: 1732223821910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:01,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:01,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:01,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:01,965 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/4acab50a0e2a43b88b648a34b25893db as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/4acab50a0e2a43b88b648a34b25893db 2024-11-21T21:16:01,971 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca977aeb13804530742b16bc2c00bf08/A of ca977aeb13804530742b16bc2c00bf08 into 4acab50a0e2a43b88b648a34b25893db(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:16:01,971 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca977aeb13804530742b16bc2c00bf08: 2024-11-21T21:16:01,972 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., storeName=ca977aeb13804530742b16bc2c00bf08/A, priority=13, startTime=1732223761131; duration=0sec 2024-11-21T21:16:01,972 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:01,972 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca977aeb13804530742b16bc2c00bf08:A 2024-11-21T21:16:01,974 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/5ad34e5b311542a0a7b21b5116bedf79 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/5ad34e5b311542a0a7b21b5116bedf79 2024-11-21T21:16:01,978 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca977aeb13804530742b16bc2c00bf08/C of ca977aeb13804530742b16bc2c00bf08 into 5ad34e5b311542a0a7b21b5116bedf79(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:16:01,978 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca977aeb13804530742b16bc2c00bf08: 2024-11-21T21:16:01,978 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., storeName=ca977aeb13804530742b16bc2c00bf08/C, priority=13, startTime=1732223761131; duration=0sec 2024-11-21T21:16:01,979 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:01,979 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca977aeb13804530742b16bc2c00bf08:C 2024-11-21T21:16:01,990 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/62f2377e4a314b9791b54c1308367e68 2024-11-21T21:16:01,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/10a24312b1fd47e39802148a4c221a7e is 50, key is test_row_0/C:col10/1732223759915/Put/seqid=0 2024-11-21T21:16:01,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742251_1427 (size=12151) 2024-11-21T21:16:02,000 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/10a24312b1fd47e39802148a4c221a7e 2024-11-21T21:16:02,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/60731f36d53947ab85d357334cb6cc54 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/60731f36d53947ab85d357334cb6cc54 2024-11-21T21:16:02,006 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/60731f36d53947ab85d357334cb6cc54, entries=150, sequenceid=250, filesize=30.4 K 2024-11-21T21:16:02,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/62f2377e4a314b9791b54c1308367e68 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/62f2377e4a314b9791b54c1308367e68 2024-11-21T21:16:02,010 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/62f2377e4a314b9791b54c1308367e68, entries=150, sequenceid=250, filesize=11.9 K 2024-11-21T21:16:02,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/10a24312b1fd47e39802148a4c221a7e as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/10a24312b1fd47e39802148a4c221a7e 2024-11-21T21:16:02,017 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/10a24312b1fd47e39802148a4c221a7e, entries=150, sequenceid=250, filesize=11.9 K 2024-11-21T21:16:02,018 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for ca977aeb13804530742b16bc2c00bf08 in 858ms, sequenceid=250, compaction requested=false 2024-11-21T21:16:02,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for ca977aeb13804530742b16bc2c00bf08: 2024-11-21T21:16:02,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:16:02,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-21T21:16:02,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-21T21:16:02,021 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-21T21:16:02,021 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7760 sec 2024-11-21T21:16:02,023 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.7810 sec 2024-11-21T21:16:02,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:16:02,128 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca977aeb13804530742b16bc2c00bf08 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-11-21T21:16:02,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=A 2024-11-21T21:16:02,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:16:02,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=B 2024-11-21T21:16:02,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:16:02,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=C 2024-11-21T21:16:02,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:16:02,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ec38421df3524bb984fc5c47e39f3d68_ca977aeb13804530742b16bc2c00bf08 is 50, key is test_row_0/A:col10/1732223762127/Put/seqid=0 2024-11-21T21:16:02,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742252_1428 (size=14994) 2024-11-21T21:16:02,143 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:16:02,146 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ec38421df3524bb984fc5c47e39f3d68_ca977aeb13804530742b16bc2c00bf08 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ec38421df3524bb984fc5c47e39f3d68_ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:16:02,147 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/f1de0b745a704752be1ab66ec7147416, store: [table=TestAcidGuarantees family=A region=ca977aeb13804530742b16bc2c00bf08] 2024-11-21T21:16:02,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/f1de0b745a704752be1ab66ec7147416 is 175, key is test_row_0/A:col10/1732223762127/Put/seqid=0 2024-11-21T21:16:02,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742253_1429 (size=39949) 2024-11-21T21:16:02,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:02,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44320 deadline: 1732223822165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:02,173 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:02,173 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:02,173 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:02,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:02,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44306 deadline: 1732223822168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:02,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:02,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:02,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:02,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:02,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44318 deadline: 1732223822230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:02,236 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:02,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:02,236 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:02,236 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:02,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44290 deadline: 1732223822230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:02,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:02,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:02,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:02,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:02,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44292 deadline: 1732223822230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:02,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:02,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:02,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:02,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:02,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44320 deadline: 1732223822280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:02,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:02,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:02,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:02,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:02,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44306 deadline: 1732223822281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:02,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:02,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:02,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:02,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-21T21:16:02,391 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:16:02,391 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:16:02,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-21T21:16:02,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-21T21:16:02,393 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:16:02,394 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:16:02,394 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:16:02,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-21T21:16:02,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:02,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44320 deadline: 1732223822501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:02,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:02,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44306 deadline: 1732223822501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:02,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:02,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:02,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:02,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:02,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:02,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:02,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-21T21:16:02,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:16:02,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. as already flushing 2024-11-21T21:16:02,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:16:02,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:02,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:02,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:02,551 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=289, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/f1de0b745a704752be1ab66ec7147416 2024-11-21T21:16:02,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/12f0325ea83a4bbf8d40ec7020c0ba34 is 50, key is test_row_0/B:col10/1732223762127/Put/seqid=0 2024-11-21T21:16:02,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742254_1430 (size=12301) 2024-11-21T21:16:02,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-21T21:16:02,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:16:02,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. as already flushing 2024-11-21T21:16:02,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:16:02,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:02,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:02,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:02,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-21T21:16:02,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:02,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44290 deadline: 1732223822740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:02,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:02,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:02,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:02,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:02,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44292 deadline: 1732223822750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:02,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:02,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:02,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:02,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44318 deadline: 1732223822750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:02,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:02,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:02,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:02,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:02,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:02,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44320 deadline: 1732223822820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:02,832 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:02,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:02,832 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:02,832 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:02,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44306 deadline: 1732223822820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:02,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:02,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:02,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:02,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-21T21:16:02,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:16:02,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. as already flushing 2024-11-21T21:16:02,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:16:02,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:02,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:02,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:02,960 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/12f0325ea83a4bbf8d40ec7020c0ba34 2024-11-21T21:16:02,968 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/103c029ff25444549819bf3d9caf50d4 is 50, key is test_row_0/C:col10/1732223762127/Put/seqid=0 2024-11-21T21:16:03,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742255_1431 (size=12301) 2024-11-21T21:16:03,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/103c029ff25444549819bf3d9caf50d4 2024-11-21T21:16:03,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/f1de0b745a704752be1ab66ec7147416 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/f1de0b745a704752be1ab66ec7147416 2024-11-21T21:16:03,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-21T21:16:03,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:16:03,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. as already flushing 2024-11-21T21:16:03,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:16:03,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:03,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:03,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:03,016 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/f1de0b745a704752be1ab66ec7147416, entries=200, sequenceid=289, filesize=39.0 K 2024-11-21T21:16:03,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/12f0325ea83a4bbf8d40ec7020c0ba34 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/12f0325ea83a4bbf8d40ec7020c0ba34 2024-11-21T21:16:03,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-21T21:16:03,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/12f0325ea83a4bbf8d40ec7020c0ba34, entries=150, sequenceid=289, filesize=12.0 K 2024-11-21T21:16:03,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/103c029ff25444549819bf3d9caf50d4 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/103c029ff25444549819bf3d9caf50d4 2024-11-21T21:16:03,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/103c029ff25444549819bf3d9caf50d4, entries=150, sequenceid=289, filesize=12.0 K 2024-11-21T21:16:03,035 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=40.25 KB/41220 for ca977aeb13804530742b16bc2c00bf08 in 907ms, sequenceid=289, compaction requested=true 2024-11-21T21:16:03,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca977aeb13804530742b16bc2c00bf08: 2024-11-21T21:16:03,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca977aeb13804530742b16bc2c00bf08:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:16:03,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:16:03,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca977aeb13804530742b16bc2c00bf08:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:16:03,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T21:16:03,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca977aeb13804530742b16bc2c00bf08:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:16:03,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-21T21:16:03,035 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:16:03,035 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:16:03,036 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:16:03,036 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): ca977aeb13804530742b16bc2c00bf08/C is initiating minor compaction (all files) 2024-11-21T21:16:03,037 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca977aeb13804530742b16bc2c00bf08/C in TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:16:03,037 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/5ad34e5b311542a0a7b21b5116bedf79, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/10a24312b1fd47e39802148a4c221a7e, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/103c029ff25444549819bf3d9caf50d4] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp, totalSize=36.3 K 2024-11-21T21:16:03,037 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102705 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:16:03,037 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): ca977aeb13804530742b16bc2c00bf08/A is initiating minor compaction (all files) 2024-11-21T21:16:03,037 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca977aeb13804530742b16bc2c00bf08/A in TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:16:03,037 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/4acab50a0e2a43b88b648a34b25893db, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/60731f36d53947ab85d357334cb6cc54, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/f1de0b745a704752be1ab66ec7147416] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp, totalSize=100.3 K 2024-11-21T21:16:03,037 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:16:03,037 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. files: [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/4acab50a0e2a43b88b648a34b25893db, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/60731f36d53947ab85d357334cb6cc54, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/f1de0b745a704752be1ab66ec7147416] 2024-11-21T21:16:03,038 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ad34e5b311542a0a7b21b5116bedf79, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732223759242 2024-11-21T21:16:03,038 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4acab50a0e2a43b88b648a34b25893db, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732223759242 2024-11-21T21:16:03,038 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 10a24312b1fd47e39802148a4c221a7e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732223759911 2024-11-21T21:16:03,038 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60731f36d53947ab85d357334cb6cc54, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732223759911 2024-11-21T21:16:03,038 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 103c029ff25444549819bf3d9caf50d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732223761566 2024-11-21T21:16:03,039 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1de0b745a704752be1ab66ec7147416, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732223761566 2024-11-21T21:16:03,047 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ca977aeb13804530742b16bc2c00bf08] 2024-11-21T21:16:03,058 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca977aeb13804530742b16bc2c00bf08#C#compaction#372 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:16:03,059 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/a1f26b10428048909aa38e9203bd0b29 is 50, key is test_row_0/C:col10/1732223762127/Put/seqid=0 2024-11-21T21:16:03,065 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121c6ef252aece24c8ab3c20ba2ebd5f3be_ca977aeb13804530742b16bc2c00bf08 store=[table=TestAcidGuarantees family=A region=ca977aeb13804530742b16bc2c00bf08] 2024-11-21T21:16:03,067 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121c6ef252aece24c8ab3c20ba2ebd5f3be_ca977aeb13804530742b16bc2c00bf08, store=[table=TestAcidGuarantees family=A region=ca977aeb13804530742b16bc2c00bf08] 2024-11-21T21:16:03,067 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c6ef252aece24c8ab3c20ba2ebd5f3be_ca977aeb13804530742b16bc2c00bf08 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ca977aeb13804530742b16bc2c00bf08] 2024-11-21T21:16:03,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742256_1432 (size=12949) 2024-11-21T21:16:03,094 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/a1f26b10428048909aa38e9203bd0b29 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/a1f26b10428048909aa38e9203bd0b29 2024-11-21T21:16:03,100 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca977aeb13804530742b16bc2c00bf08/C of ca977aeb13804530742b16bc2c00bf08 into a1f26b10428048909aa38e9203bd0b29(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:16:03,100 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca977aeb13804530742b16bc2c00bf08: 2024-11-21T21:16:03,100 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., storeName=ca977aeb13804530742b16bc2c00bf08/C, priority=13, startTime=1732223763035; duration=0sec 2024-11-21T21:16:03,100 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:16:03,100 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca977aeb13804530742b16bc2c00bf08:C 2024-11-21T21:16:03,100 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:16:03,101 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:16:03,102 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): ca977aeb13804530742b16bc2c00bf08/B is initiating minor compaction (all files) 2024-11-21T21:16:03,102 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca977aeb13804530742b16bc2c00bf08/B in TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:16:03,102 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/254fa82579a44918bff8fb96f1b72ff5, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/62f2377e4a314b9791b54c1308367e68, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/12f0325ea83a4bbf8d40ec7020c0ba34] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp, totalSize=36.3 K 2024-11-21T21:16:03,102 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 254fa82579a44918bff8fb96f1b72ff5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732223759242 2024-11-21T21:16:03,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742257_1433 (size=4469) 2024-11-21T21:16:03,105 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca977aeb13804530742b16bc2c00bf08#A#compaction#371 average throughput is 0.42 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:16:03,106 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/84dae5eaa725429b87ebaf069d1c4118 is 175, key is test_row_0/A:col10/1732223762127/Put/seqid=0 2024-11-21T21:16:03,116 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 62f2377e4a314b9791b54c1308367e68, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732223759911 2024-11-21T21:16:03,117 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 12f0325ea83a4bbf8d40ec7020c0ba34, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732223761566 2024-11-21T21:16:03,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742258_1434 (size=31903) 2024-11-21T21:16:03,129 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/84dae5eaa725429b87ebaf069d1c4118 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/84dae5eaa725429b87ebaf069d1c4118 2024-11-21T21:16:03,134 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca977aeb13804530742b16bc2c00bf08#B#compaction#373 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:16:03,135 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/634030ddeeb24ac1bd7e012415462f0c is 50, key is test_row_0/B:col10/1732223762127/Put/seqid=0 2024-11-21T21:16:03,142 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca977aeb13804530742b16bc2c00bf08/A of ca977aeb13804530742b16bc2c00bf08 into 84dae5eaa725429b87ebaf069d1c4118(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:16:03,142 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca977aeb13804530742b16bc2c00bf08: 2024-11-21T21:16:03,142 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., storeName=ca977aeb13804530742b16bc2c00bf08/A, priority=13, startTime=1732223763035; duration=0sec 2024-11-21T21:16:03,142 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:03,142 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca977aeb13804530742b16bc2c00bf08:A 2024-11-21T21:16:03,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742259_1435 (size=12949) 2024-11-21T21:16:03,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-21T21:16:03,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:16:03,162 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing ca977aeb13804530742b16bc2c00bf08 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-21T21:16:03,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=A 2024-11-21T21:16:03,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:03,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=B 2024-11-21T21:16:03,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:03,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=C 2024-11-21T21:16:03,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:03,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ddcf511f7fc940c1b8672b789edfdcae_ca977aeb13804530742b16bc2c00bf08 is 50, key is test_row_0/A:col10/1732223762130/Put/seqid=0 2024-11-21T21:16:03,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742260_1436 (size=12454) 2024-11-21T21:16:03,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:16:03,180 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ddcf511f7fc940c1b8672b789edfdcae_ca977aeb13804530742b16bc2c00bf08 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ddcf511f7fc940c1b8672b789edfdcae_ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:16:03,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/0c6d531e5e92461fabdd133770548a55, store: [table=TestAcidGuarantees family=A region=ca977aeb13804530742b16bc2c00bf08] 2024-11-21T21:16:03,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/0c6d531e5e92461fabdd133770548a55 is 175, key is test_row_0/A:col10/1732223762130/Put/seqid=0 2024-11-21T21:16:03,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742261_1437 (size=31255) 2024-11-21T21:16:03,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:16:03,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. as already flushing 2024-11-21T21:16:03,445 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732223763425, max timestamp=1732223763442Num uniques 859; 2024-11-21T21:16:03,445 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T21:16:03,445 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732223763425, max timestamp=1732223763442Num uniques 859; 2024-11-21T21:16:03,445 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732223763425, max timestamp=1732223763442Num uniques 859; 2024-11-21T21:16:03,445 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T21:16:03,445 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T21:16:03,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-21T21:16:03,536 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732223763480, max timestamp=1732223763532Num uniques 1018; 2024-11-21T21:16:03,536 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-21T21:16:03,537 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732223763480, max timestamp=1732223763532Num uniques 1018; 2024-11-21T21:16:03,537 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-21T21:16:03,540 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732223763480, max timestamp=1732223763532Num uniques 1018; 2024-11-21T21:16:03,540 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-21T21:16:03,567 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/634030ddeeb24ac1bd7e012415462f0c as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/634030ddeeb24ac1bd7e012415462f0c 2024-11-21T21:16:03,572 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca977aeb13804530742b16bc2c00bf08/B of ca977aeb13804530742b16bc2c00bf08 into 634030ddeeb24ac1bd7e012415462f0c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:16:03,572 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca977aeb13804530742b16bc2c00bf08: 2024-11-21T21:16:03,572 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., storeName=ca977aeb13804530742b16bc2c00bf08/B, priority=13, startTime=1732223763035; duration=0sec 2024-11-21T21:16:03,573 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:03,573 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca977aeb13804530742b16bc2c00bf08:B 2024-11-21T21:16:03,590 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=300, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/0c6d531e5e92461fabdd133770548a55 2024-11-21T21:16:03,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/8c663ba06ba64788aaf6c51429aa44d5 is 50, key is test_row_0/B:col10/1732223762130/Put/seqid=0 2024-11-21T21:16:03,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742262_1438 (size=12301) 2024-11-21T21:16:03,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:03,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44306 deadline: 1732223823625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:03,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:03,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:03,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:03,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:03,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44320 deadline: 1732223823638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:03,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:03,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:03,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:03,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:03,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44306 deadline: 1732223823750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:03,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:03,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:03,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:03,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:03,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44320 deadline: 1732223823750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:03,761 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:03,761 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:03,761 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:03,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:03,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44290 deadline: 1732223823760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:03,770 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,41213,1732223653469 for 'test_row_1' in ca977aeb13804530742b16bc2c00bf08 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:16:03,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:03,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:03,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:03,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:03,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44292 deadline: 1732223823770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:03,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:03,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44318 deadline: 1732223823770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:03,779 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,41213,1732223653469 for 'test_row_1' in ca977aeb13804530742b16bc2c00bf08 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:16:03,779 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,41213,1732223653469 for 'test_row_2' in ca977aeb13804530742b16bc2c00bf08 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:16:03,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:03,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:03,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:03,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:03,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:03,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:03,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:03,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44306 deadline: 1732223823970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:03,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:03,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:03,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:03,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:03,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44320 deadline: 1732223823970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:03,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:03,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:03,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:04,015 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/8c663ba06ba64788aaf6c51429aa44d5 2024-11-21T21:16:04,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/19439d0e9d92429cbfd6ff8ae9fa94ce is 50, key is test_row_0/C:col10/1732223762130/Put/seqid=0 2024-11-21T21:16:04,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742263_1439 (size=12301) 2024-11-21T21:16:04,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:04,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44306 deadline: 1732223824290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:04,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:04,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:04,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44320 deadline: 1732223824290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:04,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:04,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:04,296 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:04,296 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:04,296 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:04,424 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/19439d0e9d92429cbfd6ff8ae9fa94ce 2024-11-21T21:16:04,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/0c6d531e5e92461fabdd133770548a55 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/0c6d531e5e92461fabdd133770548a55 2024-11-21T21:16:04,435 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/0c6d531e5e92461fabdd133770548a55, entries=150, sequenceid=300, filesize=30.5 K 2024-11-21T21:16:04,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/8c663ba06ba64788aaf6c51429aa44d5 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/8c663ba06ba64788aaf6c51429aa44d5 2024-11-21T21:16:04,439 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/8c663ba06ba64788aaf6c51429aa44d5, entries=150, sequenceid=300, filesize=12.0 K 2024-11-21T21:16:04,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/19439d0e9d92429cbfd6ff8ae9fa94ce as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/19439d0e9d92429cbfd6ff8ae9fa94ce 2024-11-21T21:16:04,444 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/19439d0e9d92429cbfd6ff8ae9fa94ce, entries=150, sequenceid=300, filesize=12.0 K 2024-11-21T21:16:04,445 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for ca977aeb13804530742b16bc2c00bf08 in 1283ms, sequenceid=300, compaction requested=false 2024-11-21T21:16:04,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for ca977aeb13804530742b16bc2c00bf08: 2024-11-21T21:16:04,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:16:04,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-21T21:16:04,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-21T21:16:04,448 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-21T21:16:04,448 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0530 sec 2024-11-21T21:16:04,449 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 2.0570 sec 2024-11-21T21:16:04,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-21T21:16:04,540 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:16:04,541 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:16:04,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-21T21:16:04,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-21T21:16:04,543 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:16:04,543 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:16:04,543 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:16:04,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-21T21:16:04,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-21T21:16:04,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:16:04,695 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing ca977aeb13804530742b16bc2c00bf08 3/3 column families, dataSize=140.89 KB heapSize=399.38 KB 2024-11-21T21:16:04,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=A 2024-11-21T21:16:04,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:16:04,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=B 2024-11-21T21:16:04,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:16:04,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=C 2024-11-21T21:16:04,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:16:04,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411219c3b7d7266c74c10bb065d6276ac40b5_ca977aeb13804530742b16bc2c00bf08 is 50, key is test_row_0/A:col10/1732223763619/Put/seqid=0 2024-11-21T21:16:04,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742264_1440 (size=12454) 2024-11-21T21:16:04,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:16:04,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. as already flushing 2024-11-21T21:16:04,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:04,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44320 deadline: 1732223824826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:04,829 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:04,829 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:04,829 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:04,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:04,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44306 deadline: 1732223824827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:04,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:04,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:04,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:04,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-21T21:16:04,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:04,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44320 deadline: 1732223824940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:04,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:04,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44306 deadline: 1732223824940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:04,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:04,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:04,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:04,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:04,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:04,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:05,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:16:05,115 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411219c3b7d7266c74c10bb065d6276ac40b5_ca977aeb13804530742b16bc2c00bf08 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219c3b7d7266c74c10bb065d6276ac40b5_ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:16:05,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/a779e3b523f348c6b6c2577fc6a56b27, store: [table=TestAcidGuarantees family=A region=ca977aeb13804530742b16bc2c00bf08] 2024-11-21T21:16:05,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/a779e3b523f348c6b6c2577fc6a56b27 is 175, key is test_row_0/A:col10/1732223763619/Put/seqid=0 2024-11-21T21:16:05,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742265_1441 (size=31255) 2024-11-21T21:16:05,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:05,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44320 deadline: 1732223825160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:05,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:05,168 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:05,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44306 deadline: 1732223825160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:05,168 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:05,168 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:05,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:05,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:05,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:05,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-21T21:16:05,455 INFO [Thread-1621 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1621. 2024-11-21T21:16:05,455 DEBUG [Thread-1621 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:16:05,455 DEBUG [Thread-1621 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:05,455 DEBUG [Thread-1621 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:05,455 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:16:05,462 INFO [Thread-1627 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1627. 2024-11-21T21:16:05,462 DEBUG [Thread-1627 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:16:05,462 DEBUG [Thread-1627 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:05,462 DEBUG [Thread-1627 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:05,462 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:16:05,463 INFO [Thread-1623 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1623. 2024-11-21T21:16:05,463 DEBUG [Thread-1623 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:16:05,463 DEBUG [Thread-1623 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:05,463 DEBUG [Thread-1623 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:05,463 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:16:05,467 INFO [Thread-1625 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1625. 2024-11-21T21:16:05,467 DEBUG [Thread-1625 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:16:05,467 DEBUG [Thread-1625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:05,467 DEBUG [Thread-1625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:05,467 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:16:05,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:05,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44320 deadline: 1732223825470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:05,471 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:05,471 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:05,471 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:05,474 INFO [Thread-1619 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1619. 2024-11-21T21:16:05,474 DEBUG [Thread-1619 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:16:05,474 DEBUG [Thread-1619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:05,474 DEBUG [Thread-1619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:05,474 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:16:05,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:05,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44306 deadline: 1732223825480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:05,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:05,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:05,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:05,522 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=355, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/a779e3b523f348c6b6c2577fc6a56b27 2024-11-21T21:16:05,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/8680ceb856ca4487a332313d64427099 is 50, key is test_row_0/B:col10/1732223763619/Put/seqid=0 2024-11-21T21:16:05,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742266_1442 (size=12301) 2024-11-21T21:16:05,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-21T21:16:05,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:05,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44290 deadline: 1732223825790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:05,791 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,41213,1732223653469 for 'test_row_1' in ca977aeb13804530742b16bc2c00bf08 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:16:05,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:05,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:05,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:05,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:05,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44318 deadline: 1732223825801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:05,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:05,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44292 deadline: 1732223825801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:05,802 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,41213,1732223653469 for 'test_row_1' in ca977aeb13804530742b16bc2c00bf08 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4234 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:16:05,802 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,41213,1732223653469 for 'test_row_2' in ca977aeb13804530742b16bc2c00bf08 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:16:05,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:05,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:05,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:05,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:05,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:05,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:05,932 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/8680ceb856ca4487a332313d64427099 2024-11-21T21:16:05,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/f97a93d88c9c484c999b86cb22856e7b is 50, key is test_row_0/C:col10/1732223763619/Put/seqid=0 2024-11-21T21:16:05,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742267_1443 (size=12301) 2024-11-21T21:16:05,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44320 deadline: 1732223825981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:05,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:05,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:05,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:05,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44306 deadline: 1732223825990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:05,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:05,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:05,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:06,351 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/f97a93d88c9c484c999b86cb22856e7b 2024-11-21T21:16:06,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/a779e3b523f348c6b6c2577fc6a56b27 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/a779e3b523f348c6b6c2577fc6a56b27 2024-11-21T21:16:06,364 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/a779e3b523f348c6b6c2577fc6a56b27, entries=150, sequenceid=355, filesize=30.5 K 2024-11-21T21:16:06,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/8680ceb856ca4487a332313d64427099 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/8680ceb856ca4487a332313d64427099 2024-11-21T21:16:06,368 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/8680ceb856ca4487a332313d64427099, entries=150, sequenceid=355, filesize=12.0 K 2024-11-21T21:16:06,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/f97a93d88c9c484c999b86cb22856e7b as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/f97a93d88c9c484c999b86cb22856e7b 2024-11-21T21:16:06,374 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/f97a93d88c9c484c999b86cb22856e7b, entries=150, sequenceid=355, filesize=12.0 K 2024-11-21T21:16:06,374 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~399.33 KB/408912, currentSize=40.25 KB/41220 for ca977aeb13804530742b16bc2c00bf08 in 1679ms, sequenceid=355, compaction requested=true 2024-11-21T21:16:06,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for ca977aeb13804530742b16bc2c00bf08: 2024-11-21T21:16:06,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:16:06,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-21T21:16:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-21T21:16:06,378 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-21T21:16:06,378 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8330 sec 2024-11-21T21:16:06,381 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.8380 sec 2024-11-21T21:16:06,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-21T21:16:06,701 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:16:06,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:16:06,993 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca977aeb13804530742b16bc2c00bf08 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T21:16:06,993 INFO [Thread-1616 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1616. 2024-11-21T21:16:06,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=A 2024-11-21T21:16:06,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:06,993 DEBUG [Thread-1616 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:16:06,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=B 2024-11-21T21:16:06,993 DEBUG [Thread-1616 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:06,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:06,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=C 2024-11-21T21:16:06,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:06,993 DEBUG [Thread-1616 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:06,993 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:16:07,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411215cc9947be2fd4cb98c484baa5e210a19_ca977aeb13804530742b16bc2c00bf08 is 50, key is test_row_0/A:col10/1732223766991/Put/seqid=0 2024-11-21T21:16:07,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742268_1444 (size=12454) 2024-11-21T21:16:07,011 INFO [Thread-1608 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1608. 2024-11-21T21:16:07,011 DEBUG [Thread-1608 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:16:07,011 DEBUG [Thread-1608 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:07,012 DEBUG [Thread-1608 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:07,012 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:16:07,410 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:16:07,418 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411215cc9947be2fd4cb98c484baa5e210a19_ca977aeb13804530742b16bc2c00bf08 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215cc9947be2fd4cb98c484baa5e210a19_ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:16:07,419 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/9b8ba98145a345a58cbd5334f3cc70f1, store: [table=TestAcidGuarantees family=A region=ca977aeb13804530742b16bc2c00bf08] 2024-11-21T21:16:07,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/9b8ba98145a345a58cbd5334f3cc70f1 is 175, key is test_row_0/A:col10/1732223766991/Put/seqid=0 2024-11-21T21:16:07,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742269_1445 (size=31255) 2024-11-21T21:16:07,824 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=365, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/9b8ba98145a345a58cbd5334f3cc70f1 2024-11-21T21:16:07,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/3e6f90c420cd499da03536a6cc62059b is 50, key is test_row_0/B:col10/1732223766991/Put/seqid=0 2024-11-21T21:16:07,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742270_1446 (size=12301) 2024-11-21T21:16:08,237 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/3e6f90c420cd499da03536a6cc62059b 2024-11-21T21:16:08,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/72ba89d1af2f4ca88a0fc5c8f4db9c4f is 50, key is test_row_0/C:col10/1732223766991/Put/seqid=0 2024-11-21T21:16:08,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742271_1447 (size=12301) 2024-11-21T21:16:08,648 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/72ba89d1af2f4ca88a0fc5c8f4db9c4f 2024-11-21T21:16:08,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/9b8ba98145a345a58cbd5334f3cc70f1 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/9b8ba98145a345a58cbd5334f3cc70f1 2024-11-21T21:16:08,664 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/9b8ba98145a345a58cbd5334f3cc70f1, entries=150, sequenceid=365, filesize=30.5 K 2024-11-21T21:16:08,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/3e6f90c420cd499da03536a6cc62059b as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/3e6f90c420cd499da03536a6cc62059b 2024-11-21T21:16:08,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/3e6f90c420cd499da03536a6cc62059b, entries=150, sequenceid=365, filesize=12.0 K 2024-11-21T21:16:08,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/72ba89d1af2f4ca88a0fc5c8f4db9c4f as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/72ba89d1af2f4ca88a0fc5c8f4db9c4f 2024-11-21T21:16:08,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/72ba89d1af2f4ca88a0fc5c8f4db9c4f, entries=150, sequenceid=365, filesize=12.0 K 2024-11-21T21:16:08,676 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=6.71 KB/6870 for ca977aeb13804530742b16bc2c00bf08 in 1684ms, sequenceid=365, compaction requested=true 2024-11-21T21:16:08,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca977aeb13804530742b16bc2c00bf08: 2024-11-21T21:16:08,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca977aeb13804530742b16bc2c00bf08:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:16:08,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:08,676 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:16:08,676 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:16:08,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca977aeb13804530742b16bc2c00bf08:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:16:08,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:08,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca977aeb13804530742b16bc2c00bf08:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:16:08,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:16:08,677 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 125668 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:16:08,677 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): ca977aeb13804530742b16bc2c00bf08/A is initiating minor compaction (all files) 2024-11-21T21:16:08,677 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca977aeb13804530742b16bc2c00bf08/A in TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:16:08,677 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/84dae5eaa725429b87ebaf069d1c4118, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/0c6d531e5e92461fabdd133770548a55, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/a779e3b523f348c6b6c2577fc6a56b27, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/9b8ba98145a345a58cbd5334f3cc70f1] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp, totalSize=122.7 K 2024-11-21T21:16:08,677 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:16:08,678 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. files: [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/84dae5eaa725429b87ebaf069d1c4118, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/0c6d531e5e92461fabdd133770548a55, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/a779e3b523f348c6b6c2577fc6a56b27, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/9b8ba98145a345a58cbd5334f3cc70f1] 2024-11-21T21:16:08,678 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:16:08,678 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84dae5eaa725429b87ebaf069d1c4118, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732223761566 2024-11-21T21:16:08,678 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): ca977aeb13804530742b16bc2c00bf08/B is initiating minor compaction (all files) 2024-11-21T21:16:08,678 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca977aeb13804530742b16bc2c00bf08/B in TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:16:08,678 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/634030ddeeb24ac1bd7e012415462f0c, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/8c663ba06ba64788aaf6c51429aa44d5, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/8680ceb856ca4487a332313d64427099, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/3e6f90c420cd499da03536a6cc62059b] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp, totalSize=48.7 K 2024-11-21T21:16:08,679 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c6d531e5e92461fabdd133770548a55, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1732223762130 2024-11-21T21:16:08,679 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 634030ddeeb24ac1bd7e012415462f0c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732223761566 2024-11-21T21:16:08,679 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting a779e3b523f348c6b6c2577fc6a56b27, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732223763601 2024-11-21T21:16:08,679 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c663ba06ba64788aaf6c51429aa44d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1732223762130 2024-11-21T21:16:08,679 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b8ba98145a345a58cbd5334f3cc70f1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1732223764824 2024-11-21T21:16:08,679 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 8680ceb856ca4487a332313d64427099, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732223763601 2024-11-21T21:16:08,679 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e6f90c420cd499da03536a6cc62059b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1732223764824 2024-11-21T21:16:08,705 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ca977aeb13804530742b16bc2c00bf08] 2024-11-21T21:16:08,715 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca977aeb13804530742b16bc2c00bf08#B#compaction#384 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:16:08,716 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/09431e604ce641b1ae4eee37ccc2ce27 is 50, key is test_row_0/B:col10/1732223766991/Put/seqid=0 2024-11-21T21:16:08,725 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121c0d529d2a196493f913f0360fcc5674a_ca977aeb13804530742b16bc2c00bf08 store=[table=TestAcidGuarantees family=A region=ca977aeb13804530742b16bc2c00bf08] 2024-11-21T21:16:08,740 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121c0d529d2a196493f913f0360fcc5674a_ca977aeb13804530742b16bc2c00bf08, store=[table=TestAcidGuarantees family=A region=ca977aeb13804530742b16bc2c00bf08] 2024-11-21T21:16:08,740 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c0d529d2a196493f913f0360fcc5674a_ca977aeb13804530742b16bc2c00bf08 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ca977aeb13804530742b16bc2c00bf08] 2024-11-21T21:16:08,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742272_1448 (size=13085) 2024-11-21T21:16:08,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742273_1449 (size=4469) 2024-11-21T21:16:09,159 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/09431e604ce641b1ae4eee37ccc2ce27 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/09431e604ce641b1ae4eee37ccc2ce27 2024-11-21T21:16:09,164 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ca977aeb13804530742b16bc2c00bf08/B of ca977aeb13804530742b16bc2c00bf08 into 09431e604ce641b1ae4eee37ccc2ce27(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:16:09,164 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca977aeb13804530742b16bc2c00bf08: 2024-11-21T21:16:09,164 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., storeName=ca977aeb13804530742b16bc2c00bf08/B, priority=12, startTime=1732223768676; duration=0sec 2024-11-21T21:16:09,164 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:16:09,165 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca977aeb13804530742b16bc2c00bf08:B 2024-11-21T21:16:09,165 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:16:09,166 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:16:09,166 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): ca977aeb13804530742b16bc2c00bf08/C is initiating minor compaction (all files) 2024-11-21T21:16:09,166 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca977aeb13804530742b16bc2c00bf08/C in TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:16:09,166 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/a1f26b10428048909aa38e9203bd0b29, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/19439d0e9d92429cbfd6ff8ae9fa94ce, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/f97a93d88c9c484c999b86cb22856e7b, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/72ba89d1af2f4ca88a0fc5c8f4db9c4f] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp, totalSize=48.7 K 2024-11-21T21:16:09,166 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting a1f26b10428048909aa38e9203bd0b29, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732223761566 2024-11-21T21:16:09,167 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 19439d0e9d92429cbfd6ff8ae9fa94ce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1732223762130 2024-11-21T21:16:09,167 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting f97a93d88c9c484c999b86cb22856e7b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732223763601 2024-11-21T21:16:09,167 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 72ba89d1af2f4ca88a0fc5c8f4db9c4f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1732223764824 2024-11-21T21:16:09,175 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca977aeb13804530742b16bc2c00bf08#A#compaction#383 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:16:09,176 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/f8356d7439054f9abbb5eb1f7bbdd56e is 175, key is test_row_0/A:col10/1732223766991/Put/seqid=0 2024-11-21T21:16:09,179 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca977aeb13804530742b16bc2c00bf08#C#compaction#385 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:16:09,179 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/082188bdefe2431786e0a053725b6590 is 50, key is test_row_0/C:col10/1732223766991/Put/seqid=0 2024-11-21T21:16:09,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742275_1451 (size=13085) 2024-11-21T21:16:09,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742274_1450 (size=32039) 2024-11-21T21:16:09,202 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/f8356d7439054f9abbb5eb1f7bbdd56e as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/f8356d7439054f9abbb5eb1f7bbdd56e 2024-11-21T21:16:09,207 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ca977aeb13804530742b16bc2c00bf08/A of ca977aeb13804530742b16bc2c00bf08 into f8356d7439054f9abbb5eb1f7bbdd56e(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:16:09,207 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca977aeb13804530742b16bc2c00bf08: 2024-11-21T21:16:09,207 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., storeName=ca977aeb13804530742b16bc2c00bf08/A, priority=12, startTime=1732223768676; duration=0sec 2024-11-21T21:16:09,207 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:09,207 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca977aeb13804530742b16bc2c00bf08:A 2024-11-21T21:16:09,597 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/082188bdefe2431786e0a053725b6590 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/082188bdefe2431786e0a053725b6590 2024-11-21T21:16:09,601 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ca977aeb13804530742b16bc2c00bf08/C of ca977aeb13804530742b16bc2c00bf08 into 082188bdefe2431786e0a053725b6590(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:16:09,601 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca977aeb13804530742b16bc2c00bf08: 2024-11-21T21:16:09,601 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., storeName=ca977aeb13804530742b16bc2c00bf08/C, priority=12, startTime=1732223768676; duration=0sec 2024-11-21T21:16:09,602 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:09,602 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca977aeb13804530742b16bc2c00bf08:C 2024-11-21T21:16:09,801 INFO [Thread-1610 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1610. 2024-11-21T21:16:09,801 DEBUG [Thread-1610 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:16:09,801 DEBUG [Thread-1610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:09,801 DEBUG [Thread-1610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:09,801 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:16:09,822 INFO [Thread-1614 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1614. 2024-11-21T21:16:09,822 DEBUG [Thread-1614 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:16:09,822 DEBUG [Thread-1614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:09,822 DEBUG [Thread-1614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:09,822 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:16:09,842 INFO [Thread-1612 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1612. 2024-11-21T21:16:09,843 DEBUG [Thread-1612 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:16:09,843 DEBUG [Thread-1612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:09,843 DEBUG [Thread-1612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:09,843 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:16:09,843 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T21:16:09,843 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-11-21T21:16:09,843 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-21T21:16:09,843 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-21T21:16:09,843 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-21T21:16:09,843 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-21T21:16:09,843 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T21:16:09,843 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T21:16:09,843 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1059 2024-11-21T21:16:09,843 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3175 rows 2024-11-21T21:16:09,843 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1053 2024-11-21T21:16:09,844 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3155 rows 2024-11-21T21:16:09,844 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1052 2024-11-21T21:16:09,844 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3155 rows 2024-11-21T21:16:09,844 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1059 2024-11-21T21:16:09,844 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3173 rows 2024-11-21T21:16:09,844 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1051 2024-11-21T21:16:09,844 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3149 rows 2024-11-21T21:16:09,844 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T21:16:09,844 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T21:16:09,844 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:09,844 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:09,844 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:16:09,845 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-21T21:16:09,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T21:16:09,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-21T21:16:09,849 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732223769849"}]},"ts":"1732223769849"} 2024-11-21T21:16:09,850 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T21:16:09,850 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T21:16:09,851 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T21:16:09,855 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca977aeb13804530742b16bc2c00bf08, UNASSIGN}] 2024-11-21T21:16:09,856 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca977aeb13804530742b16bc2c00bf08, UNASSIGN 2024-11-21T21:16:09,856 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=ca977aeb13804530742b16bc2c00bf08, regionState=CLOSING, regionLocation=849dee24a242,41213,1732223653469 2024-11-21T21:16:09,858 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca977aeb13804530742b16bc2c00bf08, UNASSIGN because future has completed 2024-11-21T21:16:09,858 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T21:16:09,858 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; CloseRegionProcedure ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469}] 2024-11-21T21:16:09,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-21T21:16:10,010 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] handler.UnassignRegionHandler(122): Close ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:16:10,010 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T21:16:10,010 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1722): Closing ca977aeb13804530742b16bc2c00bf08, disabling compactions & flushes 2024-11-21T21:16:10,011 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:16:10,011 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:16:10,011 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. after waiting 0 ms 2024-11-21T21:16:10,011 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:16:10,011 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(2902): Flushing ca977aeb13804530742b16bc2c00bf08 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-21T21:16:10,011 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=A 2024-11-21T21:16:10,011 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:10,011 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=B 2024-11-21T21:16:10,011 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:10,011 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca977aeb13804530742b16bc2c00bf08, store=C 2024-11-21T21:16:10,011 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:10,017 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d0d9f7a499134277b6ee05514c51af98_ca977aeb13804530742b16bc2c00bf08 is 50, key is test_row_1/A:col10/1732223769820/Put/seqid=0 2024-11-21T21:16:10,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742276_1452 (size=9914) 2024-11-21T21:16:10,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-21T21:16:10,421 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:16:10,425 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d0d9f7a499134277b6ee05514c51af98_ca977aeb13804530742b16bc2c00bf08 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d0d9f7a499134277b6ee05514c51af98_ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:16:10,426 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/40f31cc3fa424b68a4558ca40a2d4a8b, store: [table=TestAcidGuarantees family=A region=ca977aeb13804530742b16bc2c00bf08] 2024-11-21T21:16:10,426 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/40f31cc3fa424b68a4558ca40a2d4a8b is 175, key is test_row_1/A:col10/1732223769820/Put/seqid=0 2024-11-21T21:16:10,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742277_1453 (size=22561) 2024-11-21T21:16:10,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-21T21:16:10,831 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=375, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/40f31cc3fa424b68a4558ca40a2d4a8b 2024-11-21T21:16:10,843 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/7ebaad2f17d4448b9760f10cf2e92b1d is 50, key is test_row_1/B:col10/1732223769820/Put/seqid=0 2024-11-21T21:16:10,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742278_1454 (size=9857) 2024-11-21T21:16:10,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-21T21:16:11,247 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/7ebaad2f17d4448b9760f10cf2e92b1d 2024-11-21T21:16:11,255 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/ec6a1d463d2e45d7814e1441531aed58 is 50, key is test_row_1/C:col10/1732223769820/Put/seqid=0 2024-11-21T21:16:11,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742279_1455 (size=9857) 2024-11-21T21:16:11,604 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T21:16:11,660 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/ec6a1d463d2e45d7814e1441531aed58 2024-11-21T21:16:11,671 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/A/40f31cc3fa424b68a4558ca40a2d4a8b as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/40f31cc3fa424b68a4558ca40a2d4a8b 2024-11-21T21:16:11,676 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/40f31cc3fa424b68a4558ca40a2d4a8b, entries=100, sequenceid=375, filesize=22.0 K 2024-11-21T21:16:11,676 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/B/7ebaad2f17d4448b9760f10cf2e92b1d as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/7ebaad2f17d4448b9760f10cf2e92b1d 2024-11-21T21:16:11,680 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/7ebaad2f17d4448b9760f10cf2e92b1d, entries=100, sequenceid=375, filesize=9.6 K 2024-11-21T21:16:11,681 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/.tmp/C/ec6a1d463d2e45d7814e1441531aed58 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/ec6a1d463d2e45d7814e1441531aed58 2024-11-21T21:16:11,684 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/ec6a1d463d2e45d7814e1441531aed58, entries=100, sequenceid=375, filesize=9.6 K 2024-11-21T21:16:11,685 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for ca977aeb13804530742b16bc2c00bf08 in 1674ms, sequenceid=375, compaction requested=false 2024-11-21T21:16:11,685 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/b0e262af4e5543b08fbeccb586f3098c, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/d6b4d05725ee437a98c7ae6d5440f7af, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/a7832558992d4ff3a431c952b6003571, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/59c59dd39e9e4ffabfd25aeea9fcc6f8, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/96c371fbe67d43d2bd68d6523c205231, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/5860f266f3854e90857c5d46008fcd9f, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/25b44c4d01744bbc8dbc9eadf9840fb4, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/b7f0096fc5aa4c01b6d2cc36de776428, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/4ec38aad492e45b3aba8bdb5a377e1b5, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/35dd235b6df24f0aa50268b9f63faf94, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/cef4d45b212843e18314d7cbba19ffe6, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/c3b924adfce54121bc6509fc1f8ba491, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/4915a280f99f4f158c8f7041955af399, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/f9a7e81403e646a8adde08553be7e1f4, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/51c23b6fec194c2da993853c10dcc0fb, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/67cf24bcc58c4d35934b0ddde14a3920, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/4acab50a0e2a43b88b648a34b25893db, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/60731f36d53947ab85d357334cb6cc54, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/f1de0b745a704752be1ab66ec7147416, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/84dae5eaa725429b87ebaf069d1c4118, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/0c6d531e5e92461fabdd133770548a55, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/a779e3b523f348c6b6c2577fc6a56b27, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/9b8ba98145a345a58cbd5334f3cc70f1] to archive 2024-11-21T21:16:11,686 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T21:16:11,688 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/b0e262af4e5543b08fbeccb586f3098c to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/b0e262af4e5543b08fbeccb586f3098c 2024-11-21T21:16:11,689 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/d6b4d05725ee437a98c7ae6d5440f7af to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/d6b4d05725ee437a98c7ae6d5440f7af 2024-11-21T21:16:11,690 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/a7832558992d4ff3a431c952b6003571 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/a7832558992d4ff3a431c952b6003571 2024-11-21T21:16:11,691 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/59c59dd39e9e4ffabfd25aeea9fcc6f8 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/59c59dd39e9e4ffabfd25aeea9fcc6f8 2024-11-21T21:16:11,692 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/96c371fbe67d43d2bd68d6523c205231 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/96c371fbe67d43d2bd68d6523c205231 2024-11-21T21:16:11,693 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/5860f266f3854e90857c5d46008fcd9f to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/5860f266f3854e90857c5d46008fcd9f 2024-11-21T21:16:11,694 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/25b44c4d01744bbc8dbc9eadf9840fb4 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/25b44c4d01744bbc8dbc9eadf9840fb4 2024-11-21T21:16:11,696 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/b7f0096fc5aa4c01b6d2cc36de776428 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/b7f0096fc5aa4c01b6d2cc36de776428 2024-11-21T21:16:11,697 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/4ec38aad492e45b3aba8bdb5a377e1b5 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/4ec38aad492e45b3aba8bdb5a377e1b5 2024-11-21T21:16:11,698 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/35dd235b6df24f0aa50268b9f63faf94 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/35dd235b6df24f0aa50268b9f63faf94 2024-11-21T21:16:11,700 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/cef4d45b212843e18314d7cbba19ffe6 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/cef4d45b212843e18314d7cbba19ffe6 2024-11-21T21:16:11,701 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/c3b924adfce54121bc6509fc1f8ba491 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/c3b924adfce54121bc6509fc1f8ba491 2024-11-21T21:16:11,702 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/4915a280f99f4f158c8f7041955af399 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/4915a280f99f4f158c8f7041955af399 2024-11-21T21:16:11,703 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/f9a7e81403e646a8adde08553be7e1f4 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/f9a7e81403e646a8adde08553be7e1f4 2024-11-21T21:16:11,705 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/51c23b6fec194c2da993853c10dcc0fb to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/51c23b6fec194c2da993853c10dcc0fb 2024-11-21T21:16:11,706 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/67cf24bcc58c4d35934b0ddde14a3920 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/67cf24bcc58c4d35934b0ddde14a3920 2024-11-21T21:16:11,707 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/4acab50a0e2a43b88b648a34b25893db to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/4acab50a0e2a43b88b648a34b25893db 2024-11-21T21:16:11,708 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/60731f36d53947ab85d357334cb6cc54 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/60731f36d53947ab85d357334cb6cc54 2024-11-21T21:16:11,709 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/f1de0b745a704752be1ab66ec7147416 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/f1de0b745a704752be1ab66ec7147416 2024-11-21T21:16:11,709 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/84dae5eaa725429b87ebaf069d1c4118 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/84dae5eaa725429b87ebaf069d1c4118 2024-11-21T21:16:11,710 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/0c6d531e5e92461fabdd133770548a55 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/0c6d531e5e92461fabdd133770548a55 2024-11-21T21:16:11,711 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/a779e3b523f348c6b6c2577fc6a56b27 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/a779e3b523f348c6b6c2577fc6a56b27 2024-11-21T21:16:11,712 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/9b8ba98145a345a58cbd5334f3cc70f1 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/9b8ba98145a345a58cbd5334f3cc70f1 2024-11-21T21:16:11,713 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/0842509663f843ecba27e2f1a1a2dccd, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/412f378a46eb45cb8be34e24420073b6, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/2e4c80e7a16f4408b8a7f643485ca7ca, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/02c5d888bd014d93be1b1e632d1d55b9, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/69acb7225da14e04bbd16e08e00bd894, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/74efbfadb8e745129f8539369ba18b1f, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/0f345d9862154220953bd9749d59137e, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/34e6f147e7504862a749912eb7d95e2e, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/ff681f917f8d43f890a5314ad32ce222, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/a3434d3b12c04157a1fab47eb20ffb49, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/65d90193b6a1417ba9f74cd568aca39b, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/fc1e5373558f48afa0aaccdee154cf94, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/ffc706e19a38481aa891eeb04b22fb70, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/292faf3c18be4d94a0f833de1b1887e9, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/bd53eab6b16744b482be2c1c0e513902, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/254fa82579a44918bff8fb96f1b72ff5, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/afb5d7914bf841ba9013a4296fcc8e5c, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/62f2377e4a314b9791b54c1308367e68, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/634030ddeeb24ac1bd7e012415462f0c, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/12f0325ea83a4bbf8d40ec7020c0ba34, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/8c663ba06ba64788aaf6c51429aa44d5, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/8680ceb856ca4487a332313d64427099, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/3e6f90c420cd499da03536a6cc62059b] to archive 2024-11-21T21:16:11,714 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T21:16:11,715 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/0842509663f843ecba27e2f1a1a2dccd to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/0842509663f843ecba27e2f1a1a2dccd 2024-11-21T21:16:11,715 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/412f378a46eb45cb8be34e24420073b6 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/412f378a46eb45cb8be34e24420073b6 2024-11-21T21:16:11,716 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/2e4c80e7a16f4408b8a7f643485ca7ca to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/2e4c80e7a16f4408b8a7f643485ca7ca 2024-11-21T21:16:11,717 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/02c5d888bd014d93be1b1e632d1d55b9 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/02c5d888bd014d93be1b1e632d1d55b9 2024-11-21T21:16:11,718 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/69acb7225da14e04bbd16e08e00bd894 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/69acb7225da14e04bbd16e08e00bd894 2024-11-21T21:16:11,719 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/74efbfadb8e745129f8539369ba18b1f to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/74efbfadb8e745129f8539369ba18b1f 2024-11-21T21:16:11,720 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/0f345d9862154220953bd9749d59137e to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/0f345d9862154220953bd9749d59137e 2024-11-21T21:16:11,721 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/34e6f147e7504862a749912eb7d95e2e to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/34e6f147e7504862a749912eb7d95e2e 2024-11-21T21:16:11,722 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/ff681f917f8d43f890a5314ad32ce222 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/ff681f917f8d43f890a5314ad32ce222 2024-11-21T21:16:11,723 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/a3434d3b12c04157a1fab47eb20ffb49 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/a3434d3b12c04157a1fab47eb20ffb49 2024-11-21T21:16:11,724 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/65d90193b6a1417ba9f74cd568aca39b to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/65d90193b6a1417ba9f74cd568aca39b 2024-11-21T21:16:11,725 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/fc1e5373558f48afa0aaccdee154cf94 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/fc1e5373558f48afa0aaccdee154cf94 2024-11-21T21:16:11,726 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/ffc706e19a38481aa891eeb04b22fb70 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/ffc706e19a38481aa891eeb04b22fb70 2024-11-21T21:16:11,728 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/292faf3c18be4d94a0f833de1b1887e9 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/292faf3c18be4d94a0f833de1b1887e9 2024-11-21T21:16:11,729 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/bd53eab6b16744b482be2c1c0e513902 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/bd53eab6b16744b482be2c1c0e513902 2024-11-21T21:16:11,730 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/254fa82579a44918bff8fb96f1b72ff5 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/254fa82579a44918bff8fb96f1b72ff5 2024-11-21T21:16:11,731 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/afb5d7914bf841ba9013a4296fcc8e5c to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/afb5d7914bf841ba9013a4296fcc8e5c 2024-11-21T21:16:11,732 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/62f2377e4a314b9791b54c1308367e68 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/62f2377e4a314b9791b54c1308367e68 2024-11-21T21:16:11,733 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/634030ddeeb24ac1bd7e012415462f0c to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/634030ddeeb24ac1bd7e012415462f0c 2024-11-21T21:16:11,734 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/12f0325ea83a4bbf8d40ec7020c0ba34 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/12f0325ea83a4bbf8d40ec7020c0ba34 2024-11-21T21:16:11,735 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/8c663ba06ba64788aaf6c51429aa44d5 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/8c663ba06ba64788aaf6c51429aa44d5 2024-11-21T21:16:11,736 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/8680ceb856ca4487a332313d64427099 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/8680ceb856ca4487a332313d64427099 2024-11-21T21:16:11,737 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/3e6f90c420cd499da03536a6cc62059b to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/3e6f90c420cd499da03536a6cc62059b 2024-11-21T21:16:11,739 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/8ae475fe3bc74844a1986f2d823dd505, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/d125e4864f514ce19fcecb4078cf2fd3, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/eb1b40de11db48829af46ad3e5fbc1fa, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/8edb86085f614ff08e92a374963150e7, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/45b3fb64d7a144c4917a990a9276b779, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/1d48266e64874a9d999f5ae05785c765, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/513934083c8a493baa05f0697fe636e1, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/6ad36bbe4741445d974a1733f083c8a9, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/9432d51697c641f686fe0317d66c3529, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/19c0269a9ee34b1d8341cda515c53882, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/0350ed38bbf8485f8584a6e831feb848, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/68a10e9963de45d6b6a054dc964aeba3, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/49aa739294644390bd7bab1a45d50658, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/0b5ad926b8c942aa8b5ed4c55cc3df9a, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/303ca5d0b24a481183e483180aa6d3c1, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/5ad34e5b311542a0a7b21b5116bedf79, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/d9a8980acf5741a8a15a5dcd7c56673e, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/10a24312b1fd47e39802148a4c221a7e, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/a1f26b10428048909aa38e9203bd0b29, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/103c029ff25444549819bf3d9caf50d4, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/19439d0e9d92429cbfd6ff8ae9fa94ce, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/f97a93d88c9c484c999b86cb22856e7b, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/72ba89d1af2f4ca88a0fc5c8f4db9c4f] to archive 2024-11-21T21:16:11,740 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T21:16:11,742 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/8ae475fe3bc74844a1986f2d823dd505 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/8ae475fe3bc74844a1986f2d823dd505 2024-11-21T21:16:11,743 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/d125e4864f514ce19fcecb4078cf2fd3 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/d125e4864f514ce19fcecb4078cf2fd3 2024-11-21T21:16:11,744 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/eb1b40de11db48829af46ad3e5fbc1fa to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/eb1b40de11db48829af46ad3e5fbc1fa 2024-11-21T21:16:11,746 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/8edb86085f614ff08e92a374963150e7 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/8edb86085f614ff08e92a374963150e7 2024-11-21T21:16:11,747 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/45b3fb64d7a144c4917a990a9276b779 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/45b3fb64d7a144c4917a990a9276b779 2024-11-21T21:16:11,749 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/1d48266e64874a9d999f5ae05785c765 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/1d48266e64874a9d999f5ae05785c765 2024-11-21T21:16:11,750 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/513934083c8a493baa05f0697fe636e1 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/513934083c8a493baa05f0697fe636e1 2024-11-21T21:16:11,751 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/6ad36bbe4741445d974a1733f083c8a9 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/6ad36bbe4741445d974a1733f083c8a9 2024-11-21T21:16:11,752 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/9432d51697c641f686fe0317d66c3529 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/9432d51697c641f686fe0317d66c3529 2024-11-21T21:16:11,753 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/19c0269a9ee34b1d8341cda515c53882 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/19c0269a9ee34b1d8341cda515c53882 2024-11-21T21:16:11,754 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/0350ed38bbf8485f8584a6e831feb848 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/0350ed38bbf8485f8584a6e831feb848 2024-11-21T21:16:11,755 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/68a10e9963de45d6b6a054dc964aeba3 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/68a10e9963de45d6b6a054dc964aeba3 2024-11-21T21:16:11,756 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/49aa739294644390bd7bab1a45d50658 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/49aa739294644390bd7bab1a45d50658 2024-11-21T21:16:11,757 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/0b5ad926b8c942aa8b5ed4c55cc3df9a to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/0b5ad926b8c942aa8b5ed4c55cc3df9a 2024-11-21T21:16:11,758 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/303ca5d0b24a481183e483180aa6d3c1 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/303ca5d0b24a481183e483180aa6d3c1 2024-11-21T21:16:11,759 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/5ad34e5b311542a0a7b21b5116bedf79 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/5ad34e5b311542a0a7b21b5116bedf79 2024-11-21T21:16:11,760 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/d9a8980acf5741a8a15a5dcd7c56673e to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/d9a8980acf5741a8a15a5dcd7c56673e 2024-11-21T21:16:11,761 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/10a24312b1fd47e39802148a4c221a7e to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/10a24312b1fd47e39802148a4c221a7e 2024-11-21T21:16:11,762 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/a1f26b10428048909aa38e9203bd0b29 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/a1f26b10428048909aa38e9203bd0b29 2024-11-21T21:16:11,763 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/103c029ff25444549819bf3d9caf50d4 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/103c029ff25444549819bf3d9caf50d4 2024-11-21T21:16:11,764 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/19439d0e9d92429cbfd6ff8ae9fa94ce to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/19439d0e9d92429cbfd6ff8ae9fa94ce 2024-11-21T21:16:11,765 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/f97a93d88c9c484c999b86cb22856e7b to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/f97a93d88c9c484c999b86cb22856e7b 2024-11-21T21:16:11,765 DEBUG [StoreCloser-TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/72ba89d1af2f4ca88a0fc5c8f4db9c4f to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/72ba89d1af2f4ca88a0fc5c8f4db9c4f 2024-11-21T21:16:11,770 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/recovered.edits/378.seqid, newMaxSeqId=378, maxSeqId=4 2024-11-21T21:16:11,770 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. 2024-11-21T21:16:11,770 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1676): Region close journal for ca977aeb13804530742b16bc2c00bf08: Waiting for close lock at 1732223770010Running coprocessor pre-close hooks at 1732223770010Disabling compacts and flushes for region at 1732223770010Disabling writes for close at 1732223770011 (+1 ms)Obtaining lock to block concurrent updates at 1732223770011Preparing flush snapshotting stores in ca977aeb13804530742b16bc2c00bf08 at 1732223770011Finished memstore snapshotting TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732223770011Flushing stores of TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08. at 1732223770011Flushing ca977aeb13804530742b16bc2c00bf08/A: creating writer at 1732223770012 (+1 ms)Flushing mob file ca977aeb13804530742b16bc2c00bf08/A: appending metadata at 1732223770017 (+5 ms)Flushing mob file ca977aeb13804530742b16bc2c00bf08/A: closing flushed file at 1732223770017Flushing ca977aeb13804530742b16bc2c00bf08/A: appending metadata at 1732223770426 (+409 ms)Flushing ca977aeb13804530742b16bc2c00bf08/A: closing flushed file at 1732223770426Flushing ca977aeb13804530742b16bc2c00bf08/B: creating writer at 1732223770839 (+413 ms)Flushing ca977aeb13804530742b16bc2c00bf08/B: appending metadata at 1732223770842 (+3 ms)Flushing ca977aeb13804530742b16bc2c00bf08/B: closing flushed file at 1732223770842Flushing ca977aeb13804530742b16bc2c00bf08/C: creating writer at 1732223771252 (+410 ms)Flushing ca977aeb13804530742b16bc2c00bf08/C: appending metadata at 1732223771255 (+3 ms)Flushing ca977aeb13804530742b16bc2c00bf08/C: closing flushed file at 1732223771255Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9dbcaf9: reopening flushed file at 1732223771669 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23c6611c: reopening flushed file at 1732223771676 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ddc8bcd: reopening flushed file at 1732223771680 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for ca977aeb13804530742b16bc2c00bf08 in 1674ms, sequenceid=375, compaction requested=false at 1732223771685 (+5 ms)Writing region close event to WAL at 1732223771767 (+82 ms)Running coprocessor post-close hooks at 1732223771770 (+3 ms)Closed at 1732223771770 2024-11-21T21:16:11,772 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] handler.UnassignRegionHandler(157): Closed ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:16:11,772 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=ca977aeb13804530742b16bc2c00bf08, regionState=CLOSED 2024-11-21T21:16:11,774 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=120, ppid=119, state=RUNNABLE, hasLock=false; CloseRegionProcedure ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 because future has completed 2024-11-21T21:16:11,777 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-21T21:16:11,777 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; CloseRegionProcedure ca977aeb13804530742b16bc2c00bf08, server=849dee24a242,41213,1732223653469 in 1.9170 sec 2024-11-21T21:16:11,779 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=119, resume processing ppid=118 2024-11-21T21:16:11,779 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, ppid=118, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca977aeb13804530742b16bc2c00bf08, UNASSIGN in 1.9220 sec 2024-11-21T21:16:11,781 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-21T21:16:11,781 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9280 sec 2024-11-21T21:16:11,783 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732223771783"}]},"ts":"1732223771783"} 2024-11-21T21:16:11,784 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T21:16:11,784 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T21:16:11,787 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9400 sec 2024-11-21T21:16:11,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-21T21:16:11,992 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:16:11,994 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-21T21:16:11,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:16:11,998 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=121, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:16:11,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-21T21:16:11,999 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=121, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:16:12,003 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:16:12,007 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A, FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B, FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C, FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/recovered.edits] 2024-11-21T21:16:12,010 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/40f31cc3fa424b68a4558ca40a2d4a8b to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/40f31cc3fa424b68a4558ca40a2d4a8b 2024-11-21T21:16:12,011 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/f8356d7439054f9abbb5eb1f7bbdd56e to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/A/f8356d7439054f9abbb5eb1f7bbdd56e 2024-11-21T21:16:12,013 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/09431e604ce641b1ae4eee37ccc2ce27 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/09431e604ce641b1ae4eee37ccc2ce27 2024-11-21T21:16:12,014 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/7ebaad2f17d4448b9760f10cf2e92b1d to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/B/7ebaad2f17d4448b9760f10cf2e92b1d 2024-11-21T21:16:12,016 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/082188bdefe2431786e0a053725b6590 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/082188bdefe2431786e0a053725b6590 2024-11-21T21:16:12,017 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/ec6a1d463d2e45d7814e1441531aed58 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/C/ec6a1d463d2e45d7814e1441531aed58 2024-11-21T21:16:12,020 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/recovered.edits/378.seqid to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08/recovered.edits/378.seqid 2024-11-21T21:16:12,020 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:16:12,020 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T21:16:12,021 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T21:16:12,022 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-21T21:16:12,025 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411210ed6e12e622a4422ab93745f053e202e_ca977aeb13804530742b16bc2c00bf08 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411210ed6e12e622a4422ab93745f053e202e_ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:16:12,027 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411211759110c719f4a15aaf53424769ca8ad_ca977aeb13804530742b16bc2c00bf08 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411211759110c719f4a15aaf53424769ca8ad_ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:16:12,028 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214451f375fbf94cc0a827b8b24a8cdbc1_ca977aeb13804530742b16bc2c00bf08 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214451f375fbf94cc0a827b8b24a8cdbc1_ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:16:12,029 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112145a115508d66474893b3e7d331db6468_ca977aeb13804530742b16bc2c00bf08 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112145a115508d66474893b3e7d331db6468_ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:16:12,030 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112153821b4bdfc142958564f49a00d2b3bb_ca977aeb13804530742b16bc2c00bf08 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112153821b4bdfc142958564f49a00d2b3bb_ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:16:12,030 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215906f1ca878449789ad656deafc7d4a2_ca977aeb13804530742b16bc2c00bf08 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215906f1ca878449789ad656deafc7d4a2_ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:16:12,031 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215cc9947be2fd4cb98c484baa5e210a19_ca977aeb13804530742b16bc2c00bf08 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215cc9947be2fd4cb98c484baa5e210a19_ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:16:12,032 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112161b97f4457844baeae9a773be282d923_ca977aeb13804530742b16bc2c00bf08 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112161b97f4457844baeae9a773be282d923_ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:16:12,033 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112194f6429bc5f241028e2a7c86bb7a2e36_ca977aeb13804530742b16bc2c00bf08 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112194f6429bc5f241028e2a7c86bb7a2e36_ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:16:12,033 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219b8bebd5d8ee471fbe5a9e764f980b68_ca977aeb13804530742b16bc2c00bf08 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219b8bebd5d8ee471fbe5a9e764f980b68_ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:16:12,034 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219c3b7d7266c74c10bb065d6276ac40b5_ca977aeb13804530742b16bc2c00bf08 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219c3b7d7266c74c10bb065d6276ac40b5_ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:16:12,034 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c622d47bfe2a4fe4b648863f89247c1a_ca977aeb13804530742b16bc2c00bf08 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c622d47bfe2a4fe4b648863f89247c1a_ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:16:12,035 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d0d9f7a499134277b6ee05514c51af98_ca977aeb13804530742b16bc2c00bf08 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d0d9f7a499134277b6ee05514c51af98_ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:16:12,036 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d27074ca01be427aa3db1d1152daa688_ca977aeb13804530742b16bc2c00bf08 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d27074ca01be427aa3db1d1152daa688_ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:16:12,036 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ddcf511f7fc940c1b8672b789edfdcae_ca977aeb13804530742b16bc2c00bf08 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ddcf511f7fc940c1b8672b789edfdcae_ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:16:12,037 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121de83300f12004d9cbe06d418ce29f8a2_ca977aeb13804530742b16bc2c00bf08 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121de83300f12004d9cbe06d418ce29f8a2_ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:16:12,038 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ec38421df3524bb984fc5c47e39f3d68_ca977aeb13804530742b16bc2c00bf08 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ec38421df3524bb984fc5c47e39f3d68_ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:16:12,039 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121f70c9629e2aa441db0ee9f5ef921380c_ca977aeb13804530742b16bc2c00bf08 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121f70c9629e2aa441db0ee9f5ef921380c_ca977aeb13804530742b16bc2c00bf08 2024-11-21T21:16:12,039 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T21:16:12,040 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=121, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:16:12,042 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T21:16:12,044 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T21:16:12,045 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=121, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:16:12,045 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T21:16:12,045 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732223772045"}]},"ts":"9223372036854775807"} 2024-11-21T21:16:12,046 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T21:16:12,046 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ca977aeb13804530742b16bc2c00bf08, NAME => 'TestAcidGuarantees,,1732223742714.ca977aeb13804530742b16bc2c00bf08.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T21:16:12,046 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T21:16:12,046 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732223772046"}]},"ts":"9223372036854775807"} 2024-11-21T21:16:12,049 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T21:16:12,050 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=121, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:16:12,051 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 56 msec 2024-11-21T21:16:12,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-21T21:16:12,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T21:16:12,111 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:16:12,127 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=251 (was 250) - Thread LEAK? -, OpenFileDescriptor=459 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=398 (was 392) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=11467 (was 12050) 2024-11-21T21:16:12,135 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=251, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=398, ProcessCount=11, AvailableMemoryMB=11467 2024-11-21T21:16:12,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T21:16:12,136 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T21:16:12,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=122, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T21:16:12,138 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T21:16:12,138 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:16:12,138 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 122 2024-11-21T21:16:12,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-21T21:16:12,139 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T21:16:12,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742280_1456 (size=1061) 2024-11-21T21:16:12,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-21T21:16:12,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-21T21:16:12,547 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f 2024-11-21T21:16:12,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742281_1457 (size=53) 2024-11-21T21:16:12,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-21T21:16:12,953 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:16:12,954 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 0d4d04f4516ef90780bbe9af2779af72, disabling compactions & flushes 2024-11-21T21:16:12,954 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:12,954 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:12,954 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. after waiting 0 ms 2024-11-21T21:16:12,954 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:12,954 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:12,954 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0d4d04f4516ef90780bbe9af2779af72: Waiting for close lock at 1732223772954Disabling compacts and flushes for region at 1732223772954Disabling writes for close at 1732223772954Writing region close event to WAL at 1732223772954Closed at 1732223772954 2024-11-21T21:16:12,956 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T21:16:12,956 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732223772956"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732223772956"}]},"ts":"1732223772956"} 2024-11-21T21:16:12,958 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T21:16:12,959 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T21:16:12,959 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732223772959"}]},"ts":"1732223772959"} 2024-11-21T21:16:12,961 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T21:16:12,961 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d4d04f4516ef90780bbe9af2779af72, ASSIGN}] 2024-11-21T21:16:12,963 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d4d04f4516ef90780bbe9af2779af72, ASSIGN 2024-11-21T21:16:12,964 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d4d04f4516ef90780bbe9af2779af72, ASSIGN; state=OFFLINE, location=849dee24a242,41213,1732223653469; forceNewPlan=false, retain=false 2024-11-21T21:16:13,012 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-21T21:16:13,115 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=123 updating hbase:meta row=0d4d04f4516ef90780bbe9af2779af72, regionState=OPENING, regionLocation=849dee24a242,41213,1732223653469 2024-11-21T21:16:13,117 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d4d04f4516ef90780bbe9af2779af72, ASSIGN because future has completed 2024-11-21T21:16:13,117 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469}] 2024-11-21T21:16:13,272 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:13,272 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(7752): Opening region: {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} 2024-11-21T21:16:13,272 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0d4d04f4516ef90780bbe9af2779af72 2024-11-21T21:16:13,273 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:16:13,273 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(7794): checking encryption for 0d4d04f4516ef90780bbe9af2779af72 2024-11-21T21:16:13,273 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(7797): checking classloading for 0d4d04f4516ef90780bbe9af2779af72 2024-11-21T21:16:13,274 INFO [StoreOpener-0d4d04f4516ef90780bbe9af2779af72-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0d4d04f4516ef90780bbe9af2779af72 2024-11-21T21:16:13,275 INFO [StoreOpener-0d4d04f4516ef90780bbe9af2779af72-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T21:16:13,275 INFO [StoreOpener-0d4d04f4516ef90780bbe9af2779af72-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d4d04f4516ef90780bbe9af2779af72 columnFamilyName A 2024-11-21T21:16:13,275 DEBUG [StoreOpener-0d4d04f4516ef90780bbe9af2779af72-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:16:13,276 INFO [StoreOpener-0d4d04f4516ef90780bbe9af2779af72-1 {}] regionserver.HStore(327): Store=0d4d04f4516ef90780bbe9af2779af72/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:16:13,276 INFO [StoreOpener-0d4d04f4516ef90780bbe9af2779af72-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0d4d04f4516ef90780bbe9af2779af72 2024-11-21T21:16:13,277 INFO [StoreOpener-0d4d04f4516ef90780bbe9af2779af72-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T21:16:13,277 INFO [StoreOpener-0d4d04f4516ef90780bbe9af2779af72-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d4d04f4516ef90780bbe9af2779af72 columnFamilyName B 2024-11-21T21:16:13,277 DEBUG [StoreOpener-0d4d04f4516ef90780bbe9af2779af72-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:16:13,277 INFO [StoreOpener-0d4d04f4516ef90780bbe9af2779af72-1 {}] regionserver.HStore(327): Store=0d4d04f4516ef90780bbe9af2779af72/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:16:13,277 INFO [StoreOpener-0d4d04f4516ef90780bbe9af2779af72-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0d4d04f4516ef90780bbe9af2779af72 2024-11-21T21:16:13,278 INFO [StoreOpener-0d4d04f4516ef90780bbe9af2779af72-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T21:16:13,278 INFO [StoreOpener-0d4d04f4516ef90780bbe9af2779af72-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d4d04f4516ef90780bbe9af2779af72 columnFamilyName C 2024-11-21T21:16:13,279 DEBUG [StoreOpener-0d4d04f4516ef90780bbe9af2779af72-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:16:13,279 INFO [StoreOpener-0d4d04f4516ef90780bbe9af2779af72-1 {}] regionserver.HStore(327): Store=0d4d04f4516ef90780bbe9af2779af72/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:16:13,279 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:13,279 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1038): replaying wal for 0d4d04f4516ef90780bbe9af2779af72 2024-11-21T21:16:13,280 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72 2024-11-21T21:16:13,280 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72 2024-11-21T21:16:13,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-21T21:16:13,281 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1048): stopping wal replay for 0d4d04f4516ef90780bbe9af2779af72 2024-11-21T21:16:13,281 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1060): Cleaning up temporary data for 0d4d04f4516ef90780bbe9af2779af72 2024-11-21T21:16:13,281 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T21:16:13,282 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1093): writing seq id for 0d4d04f4516ef90780bbe9af2779af72 2024-11-21T21:16:13,284 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T21:16:13,284 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1114): Opened 0d4d04f4516ef90780bbe9af2779af72; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70554287, jitterRate=0.05134080350399017}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T21:16:13,284 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0d4d04f4516ef90780bbe9af2779af72 2024-11-21T21:16:13,285 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1006): Region open journal for 0d4d04f4516ef90780bbe9af2779af72: Running coprocessor pre-open hook at 1732223773273Writing region info on filesystem at 1732223773273Initializing all the Stores at 1732223773274 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732223773274Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732223773274Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732223773274Cleaning up temporary data from old regions at 1732223773281 (+7 ms)Running coprocessor post-open hooks at 1732223773284 (+3 ms)Region opened successfully at 1732223773285 (+1 ms) 2024-11-21T21:16:13,286 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., pid=124, masterSystemTime=1732223773269 2024-11-21T21:16:13,288 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:13,288 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:13,288 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=123 updating hbase:meta row=0d4d04f4516ef90780bbe9af2779af72, regionState=OPEN, openSeqNum=2, regionLocation=849dee24a242,41213,1732223653469 2024-11-21T21:16:13,289 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=124, ppid=123, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 because future has completed 2024-11-21T21:16:13,291 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-21T21:16:13,291 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; OpenRegionProcedure 0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 in 173 msec 2024-11-21T21:16:13,292 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=123, resume processing ppid=122 2024-11-21T21:16:13,292 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, ppid=122, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d4d04f4516ef90780bbe9af2779af72, ASSIGN in 330 msec 2024-11-21T21:16:13,293 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T21:16:13,293 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732223773293"}]},"ts":"1732223773293"} 2024-11-21T21:16:13,294 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T21:16:13,295 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T21:16:13,296 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1590 sec 2024-11-21T21:16:14,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-21T21:16:14,290 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:16:14,291 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@204ac613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:14,291 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:16:14,292 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:16:14,292 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:16:14,293 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:16:14,293 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:16:14,293 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b2a5690, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:14,293 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:16:14,293 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:16:14,293 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:14,294 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49574, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:16:14,294 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17077506, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:14,295 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:16:14,295 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:16:14,295 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:16:14,296 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54336, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:16:14,297 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T21:16:14,298 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 849dee24a242,42485,1732223652547 2024-11-21T21:16:14,298 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@49937e31 2024-11-21T21:16:14,298 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T21:16:14,298 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T21:16:14,300 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74bdbad8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:14,300 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:16:14,300 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:16:14,301 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:16:14,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:16:14,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:16:14,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab169a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:14,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:16:14,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:16:14,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:14,302 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:16:14,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1466f24d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:14,303 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ac69b7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:14,303 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:16:14,304 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:16:14,304 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:16:14,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:16:14,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:16:14,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cb92783, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:14,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:16:14,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:16:14,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:14,306 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49626, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:16:14,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3812aab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:14,307 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@277a5c68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:14,307 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:16:14,307 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:16:14,308 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:16:14,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:16:14,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:16:14,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5de1044d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:14,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:16:14,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:16:14,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:14,309 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49648, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:16:14,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4183e283, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:14,310 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1df5cdc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:14,310 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:16:14,310 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:16:14,310 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:16:14,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:16:14,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:16:14,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@755ca546, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:14,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:16:14,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:16:14,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:14,312 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49662, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:16:14,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39ee0f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:14,315 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22b9b902, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:14,315 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:16:14,315 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:16:14,316 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:16:14,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:16:14,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:16:14,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15f44160, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:14,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:16:14,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:16:14,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:14,317 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49680, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:16:14,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2586745a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:14,318 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b9f4da0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:14,318 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:16:14,319 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:16:14,319 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:16:14,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:16:14,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:16:14,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64560200, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:14,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:16:14,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:16:14,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:14,320 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49700, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:16:14,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f8bea41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:14,322 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@107467c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:14,322 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:16:14,322 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:16:14,322 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:16:14,323 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:16:14,323 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:16:14,323 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57e080f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:14,323 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:16:14,323 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:16:14,323 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:14,324 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49714, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:16:14,324 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3eaa0491, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:14,325 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2acfd5e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:14,325 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:16:14,325 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:16:14,326 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:16:14,326 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:16:14,326 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:16:14,326 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d0c567c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:14,326 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:16:14,327 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:16:14,327 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:14,327 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49738, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:16:14,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a43f53f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:14,329 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@379767c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:14,329 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:16:14,329 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:16:14,330 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:16:14,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:16:14,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:16:14,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@149bb498, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:14,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:16:14,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:16:14,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:14,331 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49760, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:16:14,332 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64896f5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:14,333 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69f15306, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:14,333 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:16:14,333 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:16:14,334 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:16:14,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:16:14,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:16:14,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f01f705, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:14,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:16:14,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:16:14,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:14,335 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49782, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:16:14,336 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1710d126, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:14,337 DEBUG [Thread-2048 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:16:14,337 DEBUG [Thread-2052 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:16:14,337 DEBUG [Thread-2059 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:16:14,337 DEBUG [Thread-2063 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:16:14,337 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:16:14,337 DEBUG [Thread-2061 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:16:14,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:16:14,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:16:14,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:16:14,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:16:14,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:16:14,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:16:14,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:16:14,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:16:14,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:16:14,338 DEBUG [Thread-2046 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:16:14,339 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54340, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:16:14,339 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54342, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:16:14,339 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54356, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:16:14,339 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54354, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:16:14,339 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54368, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:16:14,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:16:14,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:16:14,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2] 2024-11-21T21:16:14,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2] 2024-11-21T21:16:14,343 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54370, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:16:14,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2] 2024-11-21T21:16:14,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2] 2024-11-21T21:16:14,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2] 2024-11-21T21:16:14,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2] 2024-11-21T21:16:14,348 DEBUG [Thread-2044 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:16:14,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:16:14,349 DEBUG [Thread-2050 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:16:14,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:16:14,350 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:16:14,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:16:14,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:16:14,351 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54376, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:16:14,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-21T21:16:14,353 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54388, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:16:14,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-21T21:16:14,353 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:16:14,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 0d4d04f4516ef90780bbe9af2779af72 2024-11-21T21:16:14,354 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:16:14,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2] 2024-11-21T21:16:14,354 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:16:14,356 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0d4d04f4516ef90780bbe9af2779af72 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T21:16:14,356 DEBUG [Thread-2055 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:16:14,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=A 2024-11-21T21:16:14,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:14,357 DEBUG [Thread-2057 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:16:14,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=B 2024-11-21T21:16:14,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:14,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=C 2024-11-21T21:16:14,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:14,357 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2] 2024-11-21T21:16:14,358 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:16:14,358 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:16:14,359 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:16:14,359 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54392, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:16:14,360 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:16:14,361 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54394, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:16:14,362 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2] 2024-11-21T21:16:14,363 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2] 2024-11-21T21:16:14,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/73910a6e4711436788fe72fb54943fde is 50, key is test_row_0/A:col10/1732223774355/Put/seqid=0 2024-11-21T21:16:14,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742282_1458 (size=12001) 2024-11-21T21:16:14,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:14,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:14,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223834396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:14,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223834391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:14,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:14,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:14,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:14,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:14,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:14,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:14,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732223774386, max timestamp=1732223774405Num uniques 859; 2024-11-21T21:16:14,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1732223774386, max timestamp=1732223774405Num uniques 859; 2024-11-21T21:16:14,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1732223774386, max timestamp=1732223774405Num uniques 859; 2024-11-21T21:16:14,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T21:16:14,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-21T21:16:14,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-21T21:16:14,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-21T21:16:14,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-21T21:16:14,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:14,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. as already flushing 2024-11-21T21:16:14,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:14,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:14,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:14,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:14,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:14,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1732223834500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:14,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:14,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:14,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:14,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54388 deadline: 1732223834500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:14,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:14,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:14,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1732223834500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:14,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:14,513 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:14,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:14,513 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:14,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:14,513 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:14,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:14,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223834540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:14,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:14,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:14,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:14,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:14,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223834540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:14,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:14,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:14,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:14,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:14,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1732223834620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:14,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:14,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:14,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1732223834620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:14,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:14,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:14,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:14,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:14,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:14,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:14,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54388 deadline: 1732223834620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:14,626 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:14,627 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:14,627 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:14,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-21T21:16:14,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:14,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. as already flushing 2024-11-21T21:16:14,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:14,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:14,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:14,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:14,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-21T21:16:14,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:14,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223834750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:14,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:14,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:14,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:14,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:14,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223834750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:14,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:14,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:14,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:14,806 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/73910a6e4711436788fe72fb54943fde 2024-11-21T21:16:14,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-21T21:16:14,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:14,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. as already flushing 2024-11-21T21:16:14,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:14,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:14,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:14,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:14,829 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/4c304c5baf2e452da39ca41f6b08663f is 50, key is test_row_0/B:col10/1732223774355/Put/seqid=0 2024-11-21T21:16:14,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742283_1459 (size=12001) 2024-11-21T21:16:14,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:14,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1732223834830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:14,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:14,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1732223834830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:14,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:14,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54388 deadline: 1732223834830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:14,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-21T21:16:14,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:14,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. as already flushing 2024-11-21T21:16:14,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:14,965 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:14,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:14,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:14,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-21T21:16:15,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:15,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223835060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:15,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:15,068 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:15,068 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:15,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223835061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:15,068 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:15,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:15,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:15,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:15,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-21T21:16:15,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:15,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. as already flushing 2024-11-21T21:16:15,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:15,118 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:15,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:15,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:15,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:15,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1732223835150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:15,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:15,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:15,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:15,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:15,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1732223835150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:15,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:15,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:15,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:15,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:15,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54388 deadline: 1732223835151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:15,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:15,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:15,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:15,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/4c304c5baf2e452da39ca41f6b08663f 2024-11-21T21:16:15,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/326106c26c9147c892b68c6f94151041 is 50, key is test_row_0/C:col10/1732223774355/Put/seqid=0 2024-11-21T21:16:15,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742284_1460 (size=12001) 2024-11-21T21:16:15,256 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/326106c26c9147c892b68c6f94151041 2024-11-21T21:16:15,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/73910a6e4711436788fe72fb54943fde as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/73910a6e4711436788fe72fb54943fde 2024-11-21T21:16:15,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/73910a6e4711436788fe72fb54943fde, entries=150, sequenceid=12, filesize=11.7 K 2024-11-21T21:16:15,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/4c304c5baf2e452da39ca41f6b08663f as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/4c304c5baf2e452da39ca41f6b08663f 2024-11-21T21:16:15,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/4c304c5baf2e452da39ca41f6b08663f, entries=150, sequenceid=12, filesize=11.7 K 2024-11-21T21:16:15,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/326106c26c9147c892b68c6f94151041 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/326106c26c9147c892b68c6f94151041 2024-11-21T21:16:15,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/326106c26c9147c892b68c6f94151041, entries=150, sequenceid=12, filesize=11.7 K 2024-11-21T21:16:15,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-21T21:16:15,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:15,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. as already flushing 2024-11-21T21:16:15,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:15,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:15,271 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=129.31 KB/132416 for 0d4d04f4516ef90780bbe9af2779af72 in 915ms, sequenceid=12, compaction requested=false 2024-11-21T21:16:15,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:15,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0d4d04f4516ef90780bbe9af2779af72: 2024-11-21T21:16:15,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:15,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-21T21:16:15,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:15,424 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 0d4d04f4516ef90780bbe9af2779af72 3/3 column families, dataSize=129.31 KB heapSize=364.54 KB 2024-11-21T21:16:15,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=A 2024-11-21T21:16:15,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:16:15,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=B 2024-11-21T21:16:15,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:16:15,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=C 2024-11-21T21:16:15,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:16:15,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/26e163cca3004dbfb43d642136325f93 is 50, key is test_row_0/A:col10/1732223774480/Put/seqid=0 2024-11-21T21:16:15,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742285_1461 (size=12001) 2024-11-21T21:16:15,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-21T21:16:15,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 0d4d04f4516ef90780bbe9af2779af72 2024-11-21T21:16:15,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. as already flushing 2024-11-21T21:16:15,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:15,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223835621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:15,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:15,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223835629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:15,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:15,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:15,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:15,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:15,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:15,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:15,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:15,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1732223835660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:15,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:15,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:15,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:15,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:15,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1732223835663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:15,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:15,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:15,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:15,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:15,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54388 deadline: 1732223835670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:15,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:15,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:15,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:15,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:15,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223835750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:15,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:15,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:15,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:15,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:15,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223835750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:15,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:15,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:15,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:15,844 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/26e163cca3004dbfb43d642136325f93 2024-11-21T21:16:15,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/6a2081e3f1964249a3f2b3f9e9c837cc is 50, key is test_row_0/B:col10/1732223774480/Put/seqid=0 2024-11-21T21:16:15,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742286_1462 (size=12001) 2024-11-21T21:16:15,855 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/6a2081e3f1964249a3f2b3f9e9c837cc 2024-11-21T21:16:15,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/7c9ecb8e6e5d4055b8bd67ec5d3ec0de is 50, key is test_row_0/C:col10/1732223774480/Put/seqid=0 2024-11-21T21:16:15,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742287_1463 (size=12001) 2024-11-21T21:16:15,871 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.86 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/7c9ecb8e6e5d4055b8bd67ec5d3ec0de 2024-11-21T21:16:15,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/26e163cca3004dbfb43d642136325f93 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/26e163cca3004dbfb43d642136325f93 2024-11-21T21:16:15,884 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/26e163cca3004dbfb43d642136325f93, entries=150, sequenceid=47, filesize=11.7 K 2024-11-21T21:16:15,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/6a2081e3f1964249a3f2b3f9e9c837cc as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/6a2081e3f1964249a3f2b3f9e9c837cc 2024-11-21T21:16:15,888 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/6a2081e3f1964249a3f2b3f9e9c837cc, entries=150, sequenceid=47, filesize=11.7 K 2024-11-21T21:16:15,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/7c9ecb8e6e5d4055b8bd67ec5d3ec0de as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/7c9ecb8e6e5d4055b8bd67ec5d3ec0de 2024-11-21T21:16:15,894 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/7c9ecb8e6e5d4055b8bd67ec5d3ec0de, entries=150, sequenceid=47, filesize=11.7 K 2024-11-21T21:16:15,895 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~129.31 KB/132416, heapSize ~364.49 KB/373240, currentSize=53.67 KB/54960 for 0d4d04f4516ef90780bbe9af2779af72 in 471ms, sequenceid=47, compaction requested=false 2024-11-21T21:16:15,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 0d4d04f4516ef90780bbe9af2779af72: 2024-11-21T21:16:15,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:15,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-21T21:16:15,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-21T21:16:15,897 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-21T21:16:15,897 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5420 sec 2024-11-21T21:16:15,900 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.5480 sec 2024-11-21T21:16:15,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 0d4d04f4516ef90780bbe9af2779af72 2024-11-21T21:16:15,976 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0d4d04f4516ef90780bbe9af2779af72 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-21T21:16:15,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=A 2024-11-21T21:16:15,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:15,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=B 2024-11-21T21:16:15,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:15,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=C 2024-11-21T21:16:15,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:16,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/5c50d054d8164287b004cdb7cbd7a2ed is 50, key is test_row_0/A:col10/1732223775973/Put/seqid=0 2024-11-21T21:16:16,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742288_1464 (size=12001) 2024-11-21T21:16:16,026 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/5c50d054d8164287b004cdb7cbd7a2ed 2024-11-21T21:16:16,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/c4f1003113a640f485b006a189aee416 is 50, key is test_row_0/B:col10/1732223775973/Put/seqid=0 2024-11-21T21:16:16,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742289_1465 (size=12001) 2024-11-21T21:16:16,060 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/c4f1003113a640f485b006a189aee416 2024-11-21T21:16:16,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/5170de82639741bcb7db9dbb513b588b is 50, key is test_row_0/C:col10/1732223775973/Put/seqid=0 2024-11-21T21:16:16,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742290_1466 (size=12001) 2024-11-21T21:16:16,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:16,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223836110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:16,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:16,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:16,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:16,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:16,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223836111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:16,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:16,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:16,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:16,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:16,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223836232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:16,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:16,242 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:16,242 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:16,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223836232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:16,242 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:16,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:16,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:16,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:16,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:16,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223836450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:16,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:16,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:16,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:16,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:16,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223836450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:16,459 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:16,459 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:16,459 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:16,500 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/5170de82639741bcb7db9dbb513b588b 2024-11-21T21:16:16,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-21T21:16:16,500 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:16:16,501 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:16:16,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-21T21:16:16,503 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:16:16,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/5c50d054d8164287b004cdb7cbd7a2ed as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/5c50d054d8164287b004cdb7cbd7a2ed 2024-11-21T21:16:16,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-21T21:16:16,505 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:16:16,505 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:16:16,509 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/5c50d054d8164287b004cdb7cbd7a2ed, entries=150, sequenceid=59, filesize=11.7 K 2024-11-21T21:16:16,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/c4f1003113a640f485b006a189aee416 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/c4f1003113a640f485b006a189aee416 2024-11-21T21:16:16,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/c4f1003113a640f485b006a189aee416, entries=150, sequenceid=59, filesize=11.7 K 2024-11-21T21:16:16,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/5170de82639741bcb7db9dbb513b588b as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/5170de82639741bcb7db9dbb513b588b 2024-11-21T21:16:16,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/5170de82639741bcb7db9dbb513b588b, entries=150, sequenceid=59, filesize=11.7 K 2024-11-21T21:16:16,524 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 0d4d04f4516ef90780bbe9af2779af72 in 548ms, sequenceid=59, compaction requested=true 2024-11-21T21:16:16,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0d4d04f4516ef90780bbe9af2779af72: 2024-11-21T21:16:16,524 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:16:16,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d4d04f4516ef90780bbe9af2779af72:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:16:16,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:16,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d4d04f4516ef90780bbe9af2779af72:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:16:16,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:16,524 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:16:16,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d4d04f4516ef90780bbe9af2779af72:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:16:16,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:16:16,525 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:16:16,525 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:16:16,525 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): 0d4d04f4516ef90780bbe9af2779af72/A is initiating minor compaction (all files) 2024-11-21T21:16:16,525 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): 0d4d04f4516ef90780bbe9af2779af72/B is initiating minor compaction (all files) 2024-11-21T21:16:16,525 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d4d04f4516ef90780bbe9af2779af72/A in TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:16,525 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d4d04f4516ef90780bbe9af2779af72/B in TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:16,525 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/73910a6e4711436788fe72fb54943fde, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/26e163cca3004dbfb43d642136325f93, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/5c50d054d8164287b004cdb7cbd7a2ed] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp, totalSize=35.2 K 2024-11-21T21:16:16,525 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/4c304c5baf2e452da39ca41f6b08663f, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/6a2081e3f1964249a3f2b3f9e9c837cc, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/c4f1003113a640f485b006a189aee416] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp, totalSize=35.2 K 2024-11-21T21:16:16,526 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73910a6e4711436788fe72fb54943fde, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732223774346 2024-11-21T21:16:16,526 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26e163cca3004dbfb43d642136325f93, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732223774466 2024-11-21T21:16:16,526 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c50d054d8164287b004cdb7cbd7a2ed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732223775604 2024-11-21T21:16:16,528 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c304c5baf2e452da39ca41f6b08663f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732223774346 2024-11-21T21:16:16,528 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a2081e3f1964249a3f2b3f9e9c837cc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732223774466 2024-11-21T21:16:16,529 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting c4f1003113a640f485b006a189aee416, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732223775604 2024-11-21T21:16:16,533 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d4d04f4516ef90780bbe9af2779af72#A#compaction#398 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:16:16,533 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/55cbccfcf4c6471cae4fd674a10f7c97 is 50, key is test_row_0/A:col10/1732223775973/Put/seqid=0 2024-11-21T21:16:16,538 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d4d04f4516ef90780bbe9af2779af72#B#compaction#399 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:16:16,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742291_1467 (size=12104) 2024-11-21T21:16:16,538 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/fbc3a0b1aa014936afe2f9c0d931a1e4 is 50, key is test_row_0/B:col10/1732223775973/Put/seqid=0 2024-11-21T21:16:16,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742292_1468 (size=12104) 2024-11-21T21:16:16,546 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/55cbccfcf4c6471cae4fd674a10f7c97 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/55cbccfcf4c6471cae4fd674a10f7c97 2024-11-21T21:16:16,550 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d4d04f4516ef90780bbe9af2779af72/A of 0d4d04f4516ef90780bbe9af2779af72 into 55cbccfcf4c6471cae4fd674a10f7c97(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:16:16,550 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d4d04f4516ef90780bbe9af2779af72: 2024-11-21T21:16:16,550 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., storeName=0d4d04f4516ef90780bbe9af2779af72/A, priority=13, startTime=1732223776524; duration=0sec 2024-11-21T21:16:16,550 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:16:16,550 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d4d04f4516ef90780bbe9af2779af72:A 2024-11-21T21:16:16,550 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:16:16,551 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:16:16,551 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): 0d4d04f4516ef90780bbe9af2779af72/C is initiating minor compaction (all files) 2024-11-21T21:16:16,551 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d4d04f4516ef90780bbe9af2779af72/C in TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:16,551 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/326106c26c9147c892b68c6f94151041, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/7c9ecb8e6e5d4055b8bd67ec5d3ec0de, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/5170de82639741bcb7db9dbb513b588b] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp, totalSize=35.2 K 2024-11-21T21:16:16,551 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 326106c26c9147c892b68c6f94151041, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732223774346 2024-11-21T21:16:16,551 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c9ecb8e6e5d4055b8bd67ec5d3ec0de, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732223774466 2024-11-21T21:16:16,552 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5170de82639741bcb7db9dbb513b588b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732223775604 2024-11-21T21:16:16,557 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d4d04f4516ef90780bbe9af2779af72#C#compaction#400 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:16:16,558 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/446cb42ef52849cdbfd90f766ec33a8a is 50, key is test_row_0/C:col10/1732223775973/Put/seqid=0 2024-11-21T21:16:16,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742293_1469 (size=12104) 2024-11-21T21:16:16,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-21T21:16:16,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-21T21:16:16,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:16,659 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 0d4d04f4516ef90780bbe9af2779af72 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-21T21:16:16,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=A 2024-11-21T21:16:16,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:16,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=B 2024-11-21T21:16:16,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:16,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=C 2024-11-21T21:16:16,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:16,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/0bdce1b80d444cbab3a8406ab2f06eee is 50, key is test_row_0/A:col10/1732223776105/Put/seqid=0 2024-11-21T21:16:16,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742294_1470 (size=12001) 2024-11-21T21:16:16,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 0d4d04f4516ef90780bbe9af2779af72 2024-11-21T21:16:16,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. as already flushing 2024-11-21T21:16:16,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:16,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54388 deadline: 1732223836710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:16,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:16,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:16,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:16,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:16,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1732223836712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:16,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:16,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:16,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1732223836716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:16,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:16,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:16,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:16,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:16,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:16,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:16,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223836771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:16,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:16,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:16,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:16,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223836771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:16,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:16,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:16,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:16,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:16,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-21T21:16:16,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:16,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54388 deadline: 1732223836820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:16,827 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:16,827 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:16,827 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:16,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:16,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1732223836830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:16,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:16,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:16,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:16,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:16,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1732223836830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:16,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:16,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:16,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:16,886 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/73910a6e4711436788fe72fb54943fde, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/26e163cca3004dbfb43d642136325f93, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/5c50d054d8164287b004cdb7cbd7a2ed] to archive 2024-11-21T21:16:16,887 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T21:16:16,888 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/73910a6e4711436788fe72fb54943fde to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/73910a6e4711436788fe72fb54943fde 2024-11-21T21:16:16,889 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/26e163cca3004dbfb43d642136325f93 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/26e163cca3004dbfb43d642136325f93 2024-11-21T21:16:16,890 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/849dee24a242:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/5c50d054d8164287b004cdb7cbd7a2ed to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/5c50d054d8164287b004cdb7cbd7a2ed 2024-11-21T21:16:16,948 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/fbc3a0b1aa014936afe2f9c0d931a1e4 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/fbc3a0b1aa014936afe2f9c0d931a1e4 2024-11-21T21:16:16,951 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d4d04f4516ef90780bbe9af2779af72/B of 0d4d04f4516ef90780bbe9af2779af72 into fbc3a0b1aa014936afe2f9c0d931a1e4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:16:16,951 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d4d04f4516ef90780bbe9af2779af72: 2024-11-21T21:16:16,951 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., storeName=0d4d04f4516ef90780bbe9af2779af72/B, priority=13, startTime=1732223776524; duration=0sec 2024-11-21T21:16:16,951 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:16,951 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d4d04f4516ef90780bbe9af2779af72:B 2024-11-21T21:16:16,967 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/446cb42ef52849cdbfd90f766ec33a8a as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/446cb42ef52849cdbfd90f766ec33a8a 2024-11-21T21:16:16,972 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d4d04f4516ef90780bbe9af2779af72/C of 0d4d04f4516ef90780bbe9af2779af72 into 446cb42ef52849cdbfd90f766ec33a8a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:16:16,972 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d4d04f4516ef90780bbe9af2779af72: 2024-11-21T21:16:16,972 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., storeName=0d4d04f4516ef90780bbe9af2779af72/C, priority=13, startTime=1732223776524; duration=0sec 2024-11-21T21:16:16,972 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:16,972 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d4d04f4516ef90780bbe9af2779af72:C 2024-11-21T21:16:17,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:17,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54388 deadline: 1732223837030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:17,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:17,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:17,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:17,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:17,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1732223837041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:17,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:17,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:17,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:17,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:17,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1732223837050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:17,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:17,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:17,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:17,066 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/0bdce1b80d444cbab3a8406ab2f06eee 2024-11-21T21:16:17,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/7854b366381a40fb99148e65458ca0ac is 50, key is test_row_0/B:col10/1732223776105/Put/seqid=0 2024-11-21T21:16:17,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742295_1471 (size=12001) 2024-11-21T21:16:17,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-21T21:16:17,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:17,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223837280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:17,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:17,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:17,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:17,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:17,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223837280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:17,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:17,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:17,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:17,290 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T21:16:17,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:17,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54388 deadline: 1732223837340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:17,347 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:17,347 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:17,347 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:17,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:17,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1732223837350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:17,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:17,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:17,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:17,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:17,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1732223837360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:17,483 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/7854b366381a40fb99148e65458ca0ac 2024-11-21T21:16:17,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/3814b52c1def4d98bcd31fa196ff7462 is 50, key is test_row_0/C:col10/1732223776105/Put/seqid=0 2024-11-21T21:16:17,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742296_1472 (size=12001) 2024-11-21T21:16:17,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-21T21:16:17,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:17,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54388 deadline: 1732223837860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:17,864 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:17,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:17,864 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:17,864 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:17,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1732223837860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:17,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:17,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:17,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:17,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:17,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1732223837880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:17,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:17,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:17,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:17,892 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/3814b52c1def4d98bcd31fa196ff7462 2024-11-21T21:16:17,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/0bdce1b80d444cbab3a8406ab2f06eee as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/0bdce1b80d444cbab3a8406ab2f06eee 2024-11-21T21:16:17,899 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/0bdce1b80d444cbab3a8406ab2f06eee, entries=150, sequenceid=80, filesize=11.7 K 2024-11-21T21:16:17,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/7854b366381a40fb99148e65458ca0ac as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/7854b366381a40fb99148e65458ca0ac 2024-11-21T21:16:17,903 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/7854b366381a40fb99148e65458ca0ac, entries=150, sequenceid=80, filesize=11.7 K 2024-11-21T21:16:17,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/3814b52c1def4d98bcd31fa196ff7462 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/3814b52c1def4d98bcd31fa196ff7462 2024-11-21T21:16:17,907 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/3814b52c1def4d98bcd31fa196ff7462, entries=150, sequenceid=80, filesize=11.7 K 2024-11-21T21:16:17,908 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 0d4d04f4516ef90780bbe9af2779af72 in 1249ms, sequenceid=80, compaction requested=false 2024-11-21T21:16:17,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 0d4d04f4516ef90780bbe9af2779af72: 2024-11-21T21:16:17,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:17,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-21T21:16:17,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-21T21:16:17,910 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-21T21:16:17,910 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4040 sec 2024-11-21T21:16:17,912 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.4100 sec 2024-11-21T21:16:18,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 0d4d04f4516ef90780bbe9af2779af72 2024-11-21T21:16:18,293 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0d4d04f4516ef90780bbe9af2779af72 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-21T21:16:18,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=A 2024-11-21T21:16:18,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:18,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=B 2024-11-21T21:16:18,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:18,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=C 2024-11-21T21:16:18,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:18,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/826a6cf5619442138b37ae34fc32afc8 is 50, key is test_row_0/A:col10/1732223776685/Put/seqid=0 2024-11-21T21:16:18,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742297_1473 (size=14341) 2024-11-21T21:16:18,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:18,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223838349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:18,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:18,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:18,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:18,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:18,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223838351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:18,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:18,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:18,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:18,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:18,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223838460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:18,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:18,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:18,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:18,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:18,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223838470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:18,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:18,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:18,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:18,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-21T21:16:18,650 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:16:18,650 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:16:18,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-21T21:16:18,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T21:16:18,652 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:16:18,653 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:16:18,653 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:16:18,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:18,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223838670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:18,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:18,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:18,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:18,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:18,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223838680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:18,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:18,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:18,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:18,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/826a6cf5619442138b37ae34fc32afc8 2024-11-21T21:16:18,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/c01b9c94d0054e7e925efecec308fa5e is 50, key is test_row_0/B:col10/1732223776685/Put/seqid=0 2024-11-21T21:16:18,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742298_1474 (size=12001) 2024-11-21T21:16:18,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T21:16:18,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-21T21:16:18,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:18,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. as already flushing 2024-11-21T21:16:18,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:18,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:18,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:18,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:18,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:18,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54388 deadline: 1732223838873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:18,878 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,41213,1732223653469 for 'test_row_1' in 0d4d04f4516ef90780bbe9af2779af72 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:16:18,879 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:18,879 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:18,879 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:18,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:18,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1732223838880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:18,885 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,41213,1732223653469 for 'test_row_2' in 0d4d04f4516ef90780bbe9af2779af72 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:16:18,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:18,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:18,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:18,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:18,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1732223838900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:18,912 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,41213,1732223653469 for 'test_row_0' in 0d4d04f4516ef90780bbe9af2779af72 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:16:18,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:18,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:18,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:18,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-21T21:16:18,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:18,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. as already flushing 2024-11-21T21:16:18,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:18,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:18,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:18,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:18,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T21:16:18,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:18,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223838980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:18,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:18,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:18,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:19,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:19,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223839000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:19,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:19,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:19,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:19,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/c01b9c94d0054e7e925efecec308fa5e 2024-11-21T21:16:19,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-21T21:16:19,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:19,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. as already flushing 2024-11-21T21:16:19,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:19,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:19,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:19,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:19,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/63a2059163314a5fa3a9dec82c953771 is 50, key is test_row_0/C:col10/1732223776685/Put/seqid=0 2024-11-21T21:16:19,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742299_1475 (size=12001) 2024-11-21T21:16:19,175 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/63a2059163314a5fa3a9dec82c953771 2024-11-21T21:16:19,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/826a6cf5619442138b37ae34fc32afc8 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/826a6cf5619442138b37ae34fc32afc8 2024-11-21T21:16:19,190 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/826a6cf5619442138b37ae34fc32afc8, entries=200, sequenceid=95, filesize=14.0 K 2024-11-21T21:16:19,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/c01b9c94d0054e7e925efecec308fa5e as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/c01b9c94d0054e7e925efecec308fa5e 2024-11-21T21:16:19,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/c01b9c94d0054e7e925efecec308fa5e, entries=150, sequenceid=95, filesize=11.7 K 2024-11-21T21:16:19,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/63a2059163314a5fa3a9dec82c953771 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/63a2059163314a5fa3a9dec82c953771 2024-11-21T21:16:19,209 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/63a2059163314a5fa3a9dec82c953771, entries=150, sequenceid=95, filesize=11.7 K 2024-11-21T21:16:19,210 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 0d4d04f4516ef90780bbe9af2779af72 in 917ms, sequenceid=95, compaction requested=true 2024-11-21T21:16:19,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0d4d04f4516ef90780bbe9af2779af72: 2024-11-21T21:16:19,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d4d04f4516ef90780bbe9af2779af72:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:16:19,211 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:16:19,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:19,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d4d04f4516ef90780bbe9af2779af72:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:16:19,211 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:16:19,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:19,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d4d04f4516ef90780bbe9af2779af72:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:16:19,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:16:19,215 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:16:19,215 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:16:19,215 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): 0d4d04f4516ef90780bbe9af2779af72/A is initiating minor compaction (all files) 2024-11-21T21:16:19,215 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): 0d4d04f4516ef90780bbe9af2779af72/B is initiating minor compaction (all files) 2024-11-21T21:16:19,215 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d4d04f4516ef90780bbe9af2779af72/A in TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:19,215 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d4d04f4516ef90780bbe9af2779af72/B in TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:19,215 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/55cbccfcf4c6471cae4fd674a10f7c97, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/0bdce1b80d444cbab3a8406ab2f06eee, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/826a6cf5619442138b37ae34fc32afc8] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp, totalSize=37.5 K 2024-11-21T21:16:19,215 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/fbc3a0b1aa014936afe2f9c0d931a1e4, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/7854b366381a40fb99148e65458ca0ac, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/c01b9c94d0054e7e925efecec308fa5e] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp, totalSize=35.3 K 2024-11-21T21:16:19,215 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting fbc3a0b1aa014936afe2f9c0d931a1e4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732223775604 2024-11-21T21:16:19,216 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55cbccfcf4c6471cae4fd674a10f7c97, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732223775604 2024-11-21T21:16:19,216 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 7854b366381a40fb99148e65458ca0ac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732223776015 2024-11-21T21:16:19,216 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bdce1b80d444cbab3a8406ab2f06eee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732223776015 2024-11-21T21:16:19,216 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 826a6cf5619442138b37ae34fc32afc8, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732223776685 2024-11-21T21:16:19,216 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting c01b9c94d0054e7e925efecec308fa5e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732223776685 2024-11-21T21:16:19,224 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d4d04f4516ef90780bbe9af2779af72#A#compaction#407 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:16:19,225 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/73299cc37b0148cbbfaaa7c82171496f is 50, key is test_row_0/A:col10/1732223776685/Put/seqid=0 2024-11-21T21:16:19,226 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d4d04f4516ef90780bbe9af2779af72#B#compaction#408 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:16:19,226 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/7adcbca81c334d8a904c8732ba128e5e is 50, key is test_row_0/B:col10/1732223776685/Put/seqid=0 2024-11-21T21:16:19,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742300_1476 (size=12104) 2024-11-21T21:16:19,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742301_1477 (size=12207) 2024-11-21T21:16:19,260 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/7adcbca81c334d8a904c8732ba128e5e as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/7adcbca81c334d8a904c8732ba128e5e 2024-11-21T21:16:19,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-21T21:16:19,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:19,271 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 0d4d04f4516ef90780bbe9af2779af72 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-21T21:16:19,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=A 2024-11-21T21:16:19,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:19,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=B 2024-11-21T21:16:19,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:19,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=C 2024-11-21T21:16:19,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:19,275 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d4d04f4516ef90780bbe9af2779af72/B of 0d4d04f4516ef90780bbe9af2779af72 into 7adcbca81c334d8a904c8732ba128e5e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:16:19,275 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d4d04f4516ef90780bbe9af2779af72: 2024-11-21T21:16:19,275 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., storeName=0d4d04f4516ef90780bbe9af2779af72/B, priority=13, startTime=1732223779211; duration=0sec 2024-11-21T21:16:19,275 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:16:19,275 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d4d04f4516ef90780bbe9af2779af72:B 2024-11-21T21:16:19,275 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:16:19,277 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:16:19,277 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): 0d4d04f4516ef90780bbe9af2779af72/C is initiating minor compaction (all files) 2024-11-21T21:16:19,277 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d4d04f4516ef90780bbe9af2779af72/C in TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:19,277 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/446cb42ef52849cdbfd90f766ec33a8a, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/3814b52c1def4d98bcd31fa196ff7462, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/63a2059163314a5fa3a9dec82c953771] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp, totalSize=35.3 K 2024-11-21T21:16:19,279 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 446cb42ef52849cdbfd90f766ec33a8a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732223775604 2024-11-21T21:16:19,279 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 3814b52c1def4d98bcd31fa196ff7462, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732223776015 2024-11-21T21:16:19,279 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 63a2059163314a5fa3a9dec82c953771, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732223776685 2024-11-21T21:16:19,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T21:16:19,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/d3fab06d0387441bb3faf2c9313f280c is 50, key is test_row_0/A:col10/1732223778348/Put/seqid=0 2024-11-21T21:16:19,305 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d4d04f4516ef90780bbe9af2779af72#C#compaction#410 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:16:19,305 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/6c9e9c739e3f4414a37ba278f8b232d2 is 50, key is test_row_0/C:col10/1732223776685/Put/seqid=0 2024-11-21T21:16:19,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742302_1478 (size=12001) 2024-11-21T21:16:19,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742303_1479 (size=12207) 2024-11-21T21:16:19,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 0d4d04f4516ef90780bbe9af2779af72 2024-11-21T21:16:19,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. as already flushing 2024-11-21T21:16:19,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:19,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223839545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:19,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:19,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:19,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:19,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223839546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:19,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:19,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:19,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:19,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:19,647 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/73299cc37b0148cbbfaaa7c82171496f as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/73299cc37b0148cbbfaaa7c82171496f 2024-11-21T21:16:19,649 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d4d04f4516ef90780bbe9af2779af72/A of 0d4d04f4516ef90780bbe9af2779af72 into 73299cc37b0148cbbfaaa7c82171496f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:16:19,649 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d4d04f4516ef90780bbe9af2779af72: 2024-11-21T21:16:19,650 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., storeName=0d4d04f4516ef90780bbe9af2779af72/A, priority=13, startTime=1732223779210; duration=0sec 2024-11-21T21:16:19,650 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:19,650 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d4d04f4516ef90780bbe9af2779af72:A 2024-11-21T21:16:19,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:19,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223839660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:19,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:19,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:19,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:19,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:19,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223839670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:19,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:19,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:19,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:19,719 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/d3fab06d0387441bb3faf2c9313f280c 2024-11-21T21:16:19,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/3d75aa6f58f34874a6e7f55a96fa5725 is 50, key is test_row_0/B:col10/1732223778348/Put/seqid=0 2024-11-21T21:16:19,725 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/6c9e9c739e3f4414a37ba278f8b232d2 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/6c9e9c739e3f4414a37ba278f8b232d2 2024-11-21T21:16:19,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742304_1480 (size=12001) 2024-11-21T21:16:19,729 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d4d04f4516ef90780bbe9af2779af72/C of 0d4d04f4516ef90780bbe9af2779af72 into 6c9e9c739e3f4414a37ba278f8b232d2(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:16:19,729 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d4d04f4516ef90780bbe9af2779af72: 2024-11-21T21:16:19,729 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., storeName=0d4d04f4516ef90780bbe9af2779af72/C, priority=13, startTime=1732223779211; duration=0sec 2024-11-21T21:16:19,730 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/3d75aa6f58f34874a6e7f55a96fa5725 2024-11-21T21:16:19,730 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:19,730 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d4d04f4516ef90780bbe9af2779af72:C 2024-11-21T21:16:19,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/62312e0341554caeadbee904c6033a34 is 50, key is test_row_0/C:col10/1732223778348/Put/seqid=0 2024-11-21T21:16:19,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742305_1481 (size=12001) 2024-11-21T21:16:19,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T21:16:19,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:19,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223839871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:19,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:19,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:19,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:19,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:19,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223839880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:19,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:19,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:19,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:20,142 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/62312e0341554caeadbee904c6033a34 2024-11-21T21:16:20,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/d3fab06d0387441bb3faf2c9313f280c as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/d3fab06d0387441bb3faf2c9313f280c 2024-11-21T21:16:20,147 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/d3fab06d0387441bb3faf2c9313f280c, entries=150, sequenceid=114, filesize=11.7 K 2024-11-21T21:16:20,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/3d75aa6f58f34874a6e7f55a96fa5725 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/3d75aa6f58f34874a6e7f55a96fa5725 2024-11-21T21:16:20,151 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/3d75aa6f58f34874a6e7f55a96fa5725, entries=150, sequenceid=114, filesize=11.7 K 2024-11-21T21:16:20,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/62312e0341554caeadbee904c6033a34 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/62312e0341554caeadbee904c6033a34 2024-11-21T21:16:20,154 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/62312e0341554caeadbee904c6033a34, entries=150, sequenceid=114, filesize=11.7 K 2024-11-21T21:16:20,155 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 0d4d04f4516ef90780bbe9af2779af72 in 883ms, sequenceid=114, compaction requested=false 2024-11-21T21:16:20,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 0d4d04f4516ef90780bbe9af2779af72: 2024-11-21T21:16:20,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:20,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-21T21:16:20,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-21T21:16:20,157 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-21T21:16:20,157 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5030 sec 2024-11-21T21:16:20,159 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.5070 sec 2024-11-21T21:16:20,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 0d4d04f4516ef90780bbe9af2779af72 2024-11-21T21:16:20,197 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0d4d04f4516ef90780bbe9af2779af72 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-21T21:16:20,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=A 2024-11-21T21:16:20,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:20,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=B 2024-11-21T21:16:20,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:20,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=C 2024-11-21T21:16:20,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:20,208 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/1c1917909d104f9f95ca0c2f0273bb5e is 50, key is test_row_0/A:col10/1732223780195/Put/seqid=0 2024-11-21T21:16:20,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742306_1482 (size=14441) 2024-11-21T21:16:20,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:20,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223840257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:20,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:20,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:20,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:20,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:20,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223840262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:20,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:20,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:20,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:20,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:20,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223840370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:20,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:20,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:20,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:20,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:20,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223840380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:20,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:20,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:20,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:20,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:20,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223840590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:20,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:20,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:20,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223840591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:20,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:20,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:20,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:20,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:20,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:20,618 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/1c1917909d104f9f95ca0c2f0273bb5e 2024-11-21T21:16:20,623 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/42578f688e8243b5a5b6fefbf2631e8d is 50, key is test_row_0/B:col10/1732223780195/Put/seqid=0 2024-11-21T21:16:20,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742307_1483 (size=12101) 2024-11-21T21:16:20,626 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/42578f688e8243b5a5b6fefbf2631e8d 2024-11-21T21:16:20,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/d30f8cfb83a94694a62ed01368d35ee5 is 50, key is test_row_0/C:col10/1732223780195/Put/seqid=0 2024-11-21T21:16:20,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742308_1484 (size=12101) 2024-11-21T21:16:20,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T21:16:20,801 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:16:20,801 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:16:20,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-21T21:16:20,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-21T21:16:20,803 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:16:20,803 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:16:20,803 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:16:20,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:20,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54388 deadline: 1732223840890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:20,897 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,41213,1732223653469 for 'test_row_1' in 0d4d04f4516ef90780bbe9af2779af72 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:16:20,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:20,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:20,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:20,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:20,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1732223840900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:20,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:20,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223840900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:20,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:20,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:20,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:20,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:20,909 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,41213,1732223653469 for 'test_row_2' in 0d4d04f4516ef90780bbe9af2779af72 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:16:20,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223840900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:20,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:20,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:20,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:20,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:20,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:20,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:20,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-21T21:16:20,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:20,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1732223840920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:20,924 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,41213,1732223653469 for 'test_row_0' in 0d4d04f4516ef90780bbe9af2779af72 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:16:20,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:20,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:20,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:20,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-21T21:16:20,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:20,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. as already flushing 2024-11-21T21:16:20,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:20,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:20,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:20,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:21,035 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/d30f8cfb83a94694a62ed01368d35ee5 2024-11-21T21:16:21,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/1c1917909d104f9f95ca0c2f0273bb5e as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/1c1917909d104f9f95ca0c2f0273bb5e 2024-11-21T21:16:21,041 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/1c1917909d104f9f95ca0c2f0273bb5e, entries=200, sequenceid=132, filesize=14.1 K 2024-11-21T21:16:21,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/42578f688e8243b5a5b6fefbf2631e8d as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/42578f688e8243b5a5b6fefbf2631e8d 2024-11-21T21:16:21,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/42578f688e8243b5a5b6fefbf2631e8d, entries=150, sequenceid=132, filesize=11.8 K 2024-11-21T21:16:21,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/d30f8cfb83a94694a62ed01368d35ee5 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/d30f8cfb83a94694a62ed01368d35ee5 2024-11-21T21:16:21,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/d30f8cfb83a94694a62ed01368d35ee5, entries=150, sequenceid=132, filesize=11.8 K 2024-11-21T21:16:21,048 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 0d4d04f4516ef90780bbe9af2779af72 in 852ms, sequenceid=132, compaction requested=true 2024-11-21T21:16:21,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0d4d04f4516ef90780bbe9af2779af72: 2024-11-21T21:16:21,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d4d04f4516ef90780bbe9af2779af72:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:16:21,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:21,048 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:16:21,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d4d04f4516ef90780bbe9af2779af72:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:16:21,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:21,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d4d04f4516ef90780bbe9af2779af72:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:16:21,048 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:16:21,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:16:21,049 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:16:21,049 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38546 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:16:21,049 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): 0d4d04f4516ef90780bbe9af2779af72/B is initiating minor compaction (all files) 2024-11-21T21:16:21,049 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): 0d4d04f4516ef90780bbe9af2779af72/A is initiating minor compaction (all files) 2024-11-21T21:16:21,049 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d4d04f4516ef90780bbe9af2779af72/B in TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:21,049 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d4d04f4516ef90780bbe9af2779af72/A in TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:21,049 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/7adcbca81c334d8a904c8732ba128e5e, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/3d75aa6f58f34874a6e7f55a96fa5725, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/42578f688e8243b5a5b6fefbf2631e8d] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp, totalSize=35.5 K 2024-11-21T21:16:21,049 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/73299cc37b0148cbbfaaa7c82171496f, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/d3fab06d0387441bb3faf2c9313f280c, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/1c1917909d104f9f95ca0c2f0273bb5e] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp, totalSize=37.6 K 2024-11-21T21:16:21,049 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 7adcbca81c334d8a904c8732ba128e5e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732223776685 2024-11-21T21:16:21,049 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73299cc37b0148cbbfaaa7c82171496f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732223776685 2024-11-21T21:16:21,049 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d75aa6f58f34874a6e7f55a96fa5725, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732223778344 2024-11-21T21:16:21,049 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3fab06d0387441bb3faf2c9313f280c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732223778344 2024-11-21T21:16:21,050 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 42578f688e8243b5a5b6fefbf2631e8d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732223779543 2024-11-21T21:16:21,050 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c1917909d104f9f95ca0c2f0273bb5e, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732223779543 2024-11-21T21:16:21,055 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d4d04f4516ef90780bbe9af2779af72#B#compaction#416 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:16:21,055 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d4d04f4516ef90780bbe9af2779af72#A#compaction#417 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:16:21,055 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/4f339f2b38c041a991ac56ad31c68906 is 50, key is test_row_0/B:col10/1732223780195/Put/seqid=0 2024-11-21T21:16:21,055 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/b2cf8c0e0e634ad7919cfb611ca633be is 50, key is test_row_0/A:col10/1732223780195/Put/seqid=0 2024-11-21T21:16:21,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742309_1485 (size=12409) 2024-11-21T21:16:21,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742310_1486 (size=12307) 2024-11-21T21:16:21,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-21T21:16:21,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:21,109 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 0d4d04f4516ef90780bbe9af2779af72 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-21T21:16:21,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=A 2024-11-21T21:16:21,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:21,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=B 2024-11-21T21:16:21,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:21,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=C 2024-11-21T21:16:21,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:21,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/c8e6d48a61644d699a0879585eaf7183 is 50, key is test_row_0/A:col10/1732223780238/Put/seqid=0 2024-11-21T21:16:21,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742311_1487 (size=12151) 2024-11-21T21:16:21,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-21T21:16:21,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 0d4d04f4516ef90780bbe9af2779af72 2024-11-21T21:16:21,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. as already flushing 2024-11-21T21:16:21,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-21T21:16:21,464 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/4f339f2b38c041a991ac56ad31c68906 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/4f339f2b38c041a991ac56ad31c68906 2024-11-21T21:16:21,465 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/b2cf8c0e0e634ad7919cfb611ca633be as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/b2cf8c0e0e634ad7919cfb611ca633be 2024-11-21T21:16:21,467 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d4d04f4516ef90780bbe9af2779af72/B of 0d4d04f4516ef90780bbe9af2779af72 into 4f339f2b38c041a991ac56ad31c68906(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:16:21,467 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d4d04f4516ef90780bbe9af2779af72: 2024-11-21T21:16:21,467 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., storeName=0d4d04f4516ef90780bbe9af2779af72/B, priority=13, startTime=1732223781048; duration=0sec 2024-11-21T21:16:21,467 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:16:21,467 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d4d04f4516ef90780bbe9af2779af72:B 2024-11-21T21:16:21,468 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:16:21,468 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:16:21,468 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d4d04f4516ef90780bbe9af2779af72/A of 0d4d04f4516ef90780bbe9af2779af72 into b2cf8c0e0e634ad7919cfb611ca633be(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:16:21,469 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): 0d4d04f4516ef90780bbe9af2779af72/C is initiating minor compaction (all files) 2024-11-21T21:16:21,469 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d4d04f4516ef90780bbe9af2779af72: 2024-11-21T21:16:21,469 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., storeName=0d4d04f4516ef90780bbe9af2779af72/A, priority=13, startTime=1732223781048; duration=0sec 2024-11-21T21:16:21,469 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d4d04f4516ef90780bbe9af2779af72/C in TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:21,469 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/6c9e9c739e3f4414a37ba278f8b232d2, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/62312e0341554caeadbee904c6033a34, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/d30f8cfb83a94694a62ed01368d35ee5] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp, totalSize=35.5 K 2024-11-21T21:16:21,469 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:21,469 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d4d04f4516ef90780bbe9af2779af72:A 2024-11-21T21:16:21,469 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c9e9c739e3f4414a37ba278f8b232d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732223776685 2024-11-21T21:16:21,469 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 62312e0341554caeadbee904c6033a34, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732223778344 2024-11-21T21:16:21,469 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting d30f8cfb83a94694a62ed01368d35ee5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732223779543 2024-11-21T21:16:21,474 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d4d04f4516ef90780bbe9af2779af72#C#compaction#419 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:16:21,474 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/f37352248ecc4dc0bc8ee343bd2e6433 is 50, key is test_row_0/C:col10/1732223780195/Put/seqid=0 2024-11-21T21:16:21,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742312_1488 (size=12409) 2024-11-21T21:16:21,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:21,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223841473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:21,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:21,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:21,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:21,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:21,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223841475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:21,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:21,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:21,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:21,517 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/c8e6d48a61644d699a0879585eaf7183 2024-11-21T21:16:21,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/25bc371dd7744957a9d751affa84d049 is 50, key is test_row_0/B:col10/1732223780238/Put/seqid=0 2024-11-21T21:16:21,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742313_1489 (size=12151) 2024-11-21T21:16:21,526 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/25bc371dd7744957a9d751affa84d049 2024-11-21T21:16:21,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/9ffd660cebf4444d94917cd364800075 is 50, key is test_row_0/C:col10/1732223780238/Put/seqid=0 2024-11-21T21:16:21,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742314_1490 (size=12151) 2024-11-21T21:16:21,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:21,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223841590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:21,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:21,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:21,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223841590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:21,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:21,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:21,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:21,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:21,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:21,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:21,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223841800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:21,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:21,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:21,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223841800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:21,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:21,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:21,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:21,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:21,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:21,881 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/f37352248ecc4dc0bc8ee343bd2e6433 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/f37352248ecc4dc0bc8ee343bd2e6433 2024-11-21T21:16:21,884 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d4d04f4516ef90780bbe9af2779af72/C of 0d4d04f4516ef90780bbe9af2779af72 into f37352248ecc4dc0bc8ee343bd2e6433(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:16:21,884 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d4d04f4516ef90780bbe9af2779af72: 2024-11-21T21:16:21,884 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., storeName=0d4d04f4516ef90780bbe9af2779af72/C, priority=13, startTime=1732223781048; duration=0sec 2024-11-21T21:16:21,885 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:21,885 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d4d04f4516ef90780bbe9af2779af72:C 2024-11-21T21:16:21,936 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/9ffd660cebf4444d94917cd364800075 2024-11-21T21:16:21,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/c8e6d48a61644d699a0879585eaf7183 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/c8e6d48a61644d699a0879585eaf7183 2024-11-21T21:16:21,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-21T21:16:21,942 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/c8e6d48a61644d699a0879585eaf7183, entries=150, sequenceid=149, filesize=11.9 K 2024-11-21T21:16:21,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/25bc371dd7744957a9d751affa84d049 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/25bc371dd7744957a9d751affa84d049 2024-11-21T21:16:21,945 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/25bc371dd7744957a9d751affa84d049, entries=150, sequenceid=149, filesize=11.9 K 2024-11-21T21:16:21,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/9ffd660cebf4444d94917cd364800075 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/9ffd660cebf4444d94917cd364800075 2024-11-21T21:16:21,948 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/9ffd660cebf4444d94917cd364800075, entries=150, sequenceid=149, filesize=11.9 K 2024-11-21T21:16:21,949 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 0d4d04f4516ef90780bbe9af2779af72 in 839ms, sequenceid=149, compaction requested=false 2024-11-21T21:16:21,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 0d4d04f4516ef90780bbe9af2779af72: 2024-11-21T21:16:21,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:21,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-21T21:16:21,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-21T21:16:21,951 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-21T21:16:21,951 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1470 sec 2024-11-21T21:16:21,953 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.1510 sec 2024-11-21T21:16:22,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 0d4d04f4516ef90780bbe9af2779af72 2024-11-21T21:16:22,112 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0d4d04f4516ef90780bbe9af2779af72 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-21T21:16:22,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=A 2024-11-21T21:16:22,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:22,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=B 2024-11-21T21:16:22,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:22,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=C 2024-11-21T21:16:22,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:22,115 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/137e20e38b2145c8bb5a4776332f6a7d is 50, key is test_row_0/A:col10/1732223782111/Put/seqid=0 2024-11-21T21:16:22,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742315_1491 (size=14541) 2024-11-21T21:16:22,123 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/137e20e38b2145c8bb5a4776332f6a7d 2024-11-21T21:16:22,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/7d64fd8d3a034b3f8c35fa28f63f9bae is 50, key is test_row_0/B:col10/1732223782111/Put/seqid=0 2024-11-21T21:16:22,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742316_1492 (size=12151) 2024-11-21T21:16:22,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:22,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223842164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:22,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:22,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:22,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:22,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223842166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:22,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:22,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:22,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:22,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:22,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:22,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223842280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:22,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:22,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:22,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:22,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:22,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223842290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:22,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:22,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:22,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:22,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:22,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223842500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:22,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:22,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:22,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:22,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223842500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:22,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:22,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:22,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:22,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:22,532 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/7d64fd8d3a034b3f8c35fa28f63f9bae 2024-11-21T21:16:22,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/541c30a7334f4729ac235868b1ca5f0f is 50, key is test_row_0/C:col10/1732223782111/Put/seqid=0 2024-11-21T21:16:22,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742317_1493 (size=12151) 2024-11-21T21:16:22,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223842820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:22,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:22,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:22,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223842820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:22,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:22,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:22,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:22,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:22,941 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/541c30a7334f4729ac235868b1ca5f0f 2024-11-21T21:16:22,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/137e20e38b2145c8bb5a4776332f6a7d as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/137e20e38b2145c8bb5a4776332f6a7d 2024-11-21T21:16:22,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/137e20e38b2145c8bb5a4776332f6a7d, entries=200, sequenceid=168, filesize=14.2 K 2024-11-21T21:16:22,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/7d64fd8d3a034b3f8c35fa28f63f9bae as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/7d64fd8d3a034b3f8c35fa28f63f9bae 2024-11-21T21:16:22,950 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/7d64fd8d3a034b3f8c35fa28f63f9bae, entries=150, sequenceid=168, filesize=11.9 K 2024-11-21T21:16:22,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-21T21:16:22,950 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:16:22,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/541c30a7334f4729ac235868b1ca5f0f as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/541c30a7334f4729ac235868b1ca5f0f 2024-11-21T21:16:22,951 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:16:22,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-21T21:16:22,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-21T21:16:22,952 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:16:22,953 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:16:22,953 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:16:22,956 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/541c30a7334f4729ac235868b1ca5f0f, entries=150, sequenceid=168, filesize=11.9 K 2024-11-21T21:16:22,957 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 0d4d04f4516ef90780bbe9af2779af72 in 845ms, sequenceid=168, compaction requested=true 2024-11-21T21:16:22,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0d4d04f4516ef90780bbe9af2779af72: 2024-11-21T21:16:22,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d4d04f4516ef90780bbe9af2779af72:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:16:22,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:22,957 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:16:22,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d4d04f4516ef90780bbe9af2779af72:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:16:22,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:22,957 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:16:22,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d4d04f4516ef90780bbe9af2779af72:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:16:22,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:16:22,958 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:16:22,958 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:16:22,958 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): 0d4d04f4516ef90780bbe9af2779af72/B is initiating minor compaction (all files) 2024-11-21T21:16:22,958 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): 0d4d04f4516ef90780bbe9af2779af72/A is initiating minor compaction (all files) 2024-11-21T21:16:22,958 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d4d04f4516ef90780bbe9af2779af72/B in TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:22,958 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d4d04f4516ef90780bbe9af2779af72/A in TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:22,958 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/4f339f2b38c041a991ac56ad31c68906, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/25bc371dd7744957a9d751affa84d049, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/7d64fd8d3a034b3f8c35fa28f63f9bae] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp, totalSize=35.9 K 2024-11-21T21:16:22,958 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/b2cf8c0e0e634ad7919cfb611ca633be, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/c8e6d48a61644d699a0879585eaf7183, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/137e20e38b2145c8bb5a4776332f6a7d] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp, totalSize=38.1 K 2024-11-21T21:16:22,958 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f339f2b38c041a991ac56ad31c68906, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732223779543 2024-11-21T21:16:22,958 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2cf8c0e0e634ad7919cfb611ca633be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732223779543 2024-11-21T21:16:22,958 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 25bc371dd7744957a9d751affa84d049, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732223780224 2024-11-21T21:16:22,958 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8e6d48a61644d699a0879585eaf7183, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732223780224 2024-11-21T21:16:22,959 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d64fd8d3a034b3f8c35fa28f63f9bae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732223781460 2024-11-21T21:16:22,959 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 137e20e38b2145c8bb5a4776332f6a7d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732223781460 2024-11-21T21:16:22,963 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d4d04f4516ef90780bbe9af2779af72#B#compaction#425 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:16:22,963 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d4d04f4516ef90780bbe9af2779af72#A#compaction#426 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:16:22,963 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/b0d82f917cb045929a326078e6b44487 is 50, key is test_row_0/A:col10/1732223782111/Put/seqid=0 2024-11-21T21:16:22,963 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/a70deeb45a1344e1a51d9dec97580c06 is 50, key is test_row_0/B:col10/1732223782111/Put/seqid=0 2024-11-21T21:16:22,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742318_1494 (size=12561) 2024-11-21T21:16:22,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742319_1495 (size=12459) 2024-11-21T21:16:23,010 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-21T21:16:23,010 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-21T21:16:23,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-21T21:16:23,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-21T21:16:23,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:23,105 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 0d4d04f4516ef90780bbe9af2779af72 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-21T21:16:23,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=A 2024-11-21T21:16:23,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:23,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=B 2024-11-21T21:16:23,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:23,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=C 2024-11-21T21:16:23,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:23,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/d4c4b16a680d4e9cac7d0cc51224cee9 is 50, key is test_row_0/A:col10/1732223782163/Put/seqid=0 2024-11-21T21:16:23,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742320_1496 (size=12151) 2024-11-21T21:16:23,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-21T21:16:23,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 0d4d04f4516ef90780bbe9af2779af72 2024-11-21T21:16:23,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. as already flushing 2024-11-21T21:16:23,370 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/a70deeb45a1344e1a51d9dec97580c06 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/a70deeb45a1344e1a51d9dec97580c06 2024-11-21T21:16:23,373 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d4d04f4516ef90780bbe9af2779af72/B of 0d4d04f4516ef90780bbe9af2779af72 into a70deeb45a1344e1a51d9dec97580c06(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:16:23,373 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d4d04f4516ef90780bbe9af2779af72: 2024-11-21T21:16:23,374 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., storeName=0d4d04f4516ef90780bbe9af2779af72/B, priority=13, startTime=1732223782957; duration=0sec 2024-11-21T21:16:23,374 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:16:23,374 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d4d04f4516ef90780bbe9af2779af72:B 2024-11-21T21:16:23,374 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:16:23,374 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:16:23,374 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): 0d4d04f4516ef90780bbe9af2779af72/C is initiating minor compaction (all files) 2024-11-21T21:16:23,374 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d4d04f4516ef90780bbe9af2779af72/C in TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:23,375 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/f37352248ecc4dc0bc8ee343bd2e6433, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/9ffd660cebf4444d94917cd364800075, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/541c30a7334f4729ac235868b1ca5f0f] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp, totalSize=35.9 K 2024-11-21T21:16:23,375 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting f37352248ecc4dc0bc8ee343bd2e6433, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732223779543 2024-11-21T21:16:23,375 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ffd660cebf4444d94917cd364800075, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732223780224 2024-11-21T21:16:23,375 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 541c30a7334f4729ac235868b1ca5f0f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732223781460 2024-11-21T21:16:23,376 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/b0d82f917cb045929a326078e6b44487 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/b0d82f917cb045929a326078e6b44487 2024-11-21T21:16:23,380 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d4d04f4516ef90780bbe9af2779af72/A of 0d4d04f4516ef90780bbe9af2779af72 into b0d82f917cb045929a326078e6b44487(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:16:23,380 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d4d04f4516ef90780bbe9af2779af72: 2024-11-21T21:16:23,381 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., storeName=0d4d04f4516ef90780bbe9af2779af72/A, priority=13, startTime=1732223782957; duration=0sec 2024-11-21T21:16:23,381 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:23,381 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d4d04f4516ef90780bbe9af2779af72:A 2024-11-21T21:16:23,382 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d4d04f4516ef90780bbe9af2779af72#C#compaction#428 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:16:23,382 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/b1c50f70f10741a0a1f13d57c8377002 is 50, key is test_row_0/C:col10/1732223782111/Put/seqid=0 2024-11-21T21:16:23,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742321_1497 (size=12561) 2024-11-21T21:16:23,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:23,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223843378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:23,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:23,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:23,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:23,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:23,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223843380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:23,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:23,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:23,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:23,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:23,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223843490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:23,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:23,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:23,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:23,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:23,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223843500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:23,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:23,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:23,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:23,513 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/d4c4b16a680d4e9cac7d0cc51224cee9 2024-11-21T21:16:23,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/81e99b7c9812431795b2b4c8d830d1dc is 50, key is test_row_0/B:col10/1732223782163/Put/seqid=0 2024-11-21T21:16:23,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742322_1498 (size=12151) 2024-11-21T21:16:23,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-21T21:16:23,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:23,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223843700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:23,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:23,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:23,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:23,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:23,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223843710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:23,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:23,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:23,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:23,788 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/b1c50f70f10741a0a1f13d57c8377002 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/b1c50f70f10741a0a1f13d57c8377002 2024-11-21T21:16:23,792 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d4d04f4516ef90780bbe9af2779af72/C of 0d4d04f4516ef90780bbe9af2779af72 into b1c50f70f10741a0a1f13d57c8377002(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:16:23,792 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d4d04f4516ef90780bbe9af2779af72: 2024-11-21T21:16:23,792 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., storeName=0d4d04f4516ef90780bbe9af2779af72/C, priority=13, startTime=1732223782957; duration=0sec 2024-11-21T21:16:23,792 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:23,792 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d4d04f4516ef90780bbe9af2779af72:C 2024-11-21T21:16:23,931 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/81e99b7c9812431795b2b4c8d830d1dc 2024-11-21T21:16:23,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/43ce028d9c9d4411a00d9d4c7b54cbb1 is 50, key is test_row_0/C:col10/1732223782163/Put/seqid=0 2024-11-21T21:16:23,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742323_1499 (size=12151) 2024-11-21T21:16:23,944 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/43ce028d9c9d4411a00d9d4c7b54cbb1 2024-11-21T21:16:23,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/d4c4b16a680d4e9cac7d0cc51224cee9 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/d4c4b16a680d4e9cac7d0cc51224cee9 2024-11-21T21:16:23,951 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/d4c4b16a680d4e9cac7d0cc51224cee9, entries=150, sequenceid=184, filesize=11.9 K 2024-11-21T21:16:23,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/81e99b7c9812431795b2b4c8d830d1dc as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/81e99b7c9812431795b2b4c8d830d1dc 2024-11-21T21:16:23,955 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/81e99b7c9812431795b2b4c8d830d1dc, entries=150, sequenceid=184, filesize=11.9 K 2024-11-21T21:16:23,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/43ce028d9c9d4411a00d9d4c7b54cbb1 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/43ce028d9c9d4411a00d9d4c7b54cbb1 2024-11-21T21:16:23,957 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/43ce028d9c9d4411a00d9d4c7b54cbb1, entries=150, sequenceid=184, filesize=11.9 K 2024-11-21T21:16:23,958 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 0d4d04f4516ef90780bbe9af2779af72 in 853ms, sequenceid=184, compaction requested=false 2024-11-21T21:16:23,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 0d4d04f4516ef90780bbe9af2779af72: 2024-11-21T21:16:23,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:23,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-21T21:16:23,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-21T21:16:23,960 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-21T21:16:23,960 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0060 sec 2024-11-21T21:16:23,962 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.0100 sec 2024-11-21T21:16:24,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 0d4d04f4516ef90780bbe9af2779af72 2024-11-21T21:16:24,013 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0d4d04f4516ef90780bbe9af2779af72 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-21T21:16:24,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=A 2024-11-21T21:16:24,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:24,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=B 2024-11-21T21:16:24,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:24,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=C 2024-11-21T21:16:24,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:24,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/8264820331154d49865b560580514a3f is 50, key is test_row_0/A:col10/1732223784012/Put/seqid=0 2024-11-21T21:16:24,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742324_1500 (size=14541) 2024-11-21T21:16:24,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:24,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223844064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:24,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:24,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:24,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:24,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:24,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223844069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:24,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:24,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:24,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:24,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-21T21:16:24,091 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:16:24,091 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:16:24,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-21T21:16:24,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-21T21:16:24,093 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:16:24,093 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:16:24,093 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:16:24,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:24,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223844180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:24,188 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:24,188 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:24,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:24,188 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:24,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223844180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:24,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:24,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:24,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:24,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-21T21:16:24,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-21T21:16:24,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:24,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. as already flushing 2024-11-21T21:16:24,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:24,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:24,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:24,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:24,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:24,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223844390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:24,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:24,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:24,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:24,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-21T21:16:24,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:24,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. as already flushing 2024-11-21T21:16:24,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:24,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:24,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:24,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:24,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:24,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223844400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:24,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:24,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:24,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:24,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-21T21:16:24,421 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/8264820331154d49865b560580514a3f 2024-11-21T21:16:24,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/c091938486fe46529338d07f8e95834b is 50, key is test_row_0/B:col10/1732223784012/Put/seqid=0 2024-11-21T21:16:24,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742325_1501 (size=12151) 2024-11-21T21:16:24,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-21T21:16:24,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:24,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. as already flushing 2024-11-21T21:16:24,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:24,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:24,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:24,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:24,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-21T21:16:24,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:24,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. as already flushing 2024-11-21T21:16:24,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:24,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:24,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:24,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:24,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:24,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223844700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:24,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:24,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:24,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:24,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:24,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223844710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:24,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:24,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:24,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:24,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-21T21:16:24,831 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/c091938486fe46529338d07f8e95834b 2024-11-21T21:16:24,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/cc99c46375ff4eafa776dbb8f6988d75 is 50, key is test_row_0/C:col10/1732223784012/Put/seqid=0 2024-11-21T21:16:24,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742326_1502 (size=12151) 2024-11-21T21:16:24,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-21T21:16:24,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:24,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. as already flushing 2024-11-21T21:16:24,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:24,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:24,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:24,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:24,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:24,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54388 deadline: 1732223844910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:24,916 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,41213,1732223653469 for 'test_row_1' in 0d4d04f4516ef90780bbe9af2779af72 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:16:24,917 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:24,917 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:24,917 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:24,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:24,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54354 deadline: 1732223844920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:24,928 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,41213,1732223653469 for 'test_row_2' in 0d4d04f4516ef90780bbe9af2779af72 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:16:24,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:24,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:24,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:24,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:24,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1732223844940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:24,949 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,41213,1732223653469 for 'test_row_0' in 0d4d04f4516ef90780bbe9af2779af72 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:16:24,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:24,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:24,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:25,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-21T21:16:25,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:25,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. as already flushing 2024-11-21T21:16:25,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:25,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:25,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:25,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:25,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-21T21:16:25,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:25,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. as already flushing 2024-11-21T21:16:25,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:25,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:25,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:25,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:25,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:25,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223845220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:25,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:25,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:25,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:25,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-21T21:16:25,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:25,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223845230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:25,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:25,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:25,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:25,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/cc99c46375ff4eafa776dbb8f6988d75 2024-11-21T21:16:25,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/8264820331154d49865b560580514a3f as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/8264820331154d49865b560580514a3f 2024-11-21T21:16:25,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/8264820331154d49865b560580514a3f, entries=200, sequenceid=204, filesize=14.2 K 2024-11-21T21:16:25,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/c091938486fe46529338d07f8e95834b as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/c091938486fe46529338d07f8e95834b 2024-11-21T21:16:25,248 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/c091938486fe46529338d07f8e95834b, entries=150, sequenceid=204, filesize=11.9 K 2024-11-21T21:16:25,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/cc99c46375ff4eafa776dbb8f6988d75 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/cc99c46375ff4eafa776dbb8f6988d75 2024-11-21T21:16:25,251 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/cc99c46375ff4eafa776dbb8f6988d75, entries=150, sequenceid=204, filesize=11.9 K 2024-11-21T21:16:25,252 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 0d4d04f4516ef90780bbe9af2779af72 in 1238ms, sequenceid=204, compaction requested=true 2024-11-21T21:16:25,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0d4d04f4516ef90780bbe9af2779af72: 2024-11-21T21:16:25,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d4d04f4516ef90780bbe9af2779af72:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:16:25,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:25,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d4d04f4516ef90780bbe9af2779af72:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:16:25,252 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:16:25,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:25,252 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:16:25,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d4d04f4516ef90780bbe9af2779af72:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:16:25,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:16:25,252 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39151 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:16:25,252 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:16:25,252 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): 0d4d04f4516ef90780bbe9af2779af72/A is initiating minor compaction (all files) 2024-11-21T21:16:25,252 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): 0d4d04f4516ef90780bbe9af2779af72/B is initiating minor compaction (all files) 2024-11-21T21:16:25,252 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d4d04f4516ef90780bbe9af2779af72/A in TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:25,252 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d4d04f4516ef90780bbe9af2779af72/B in TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:25,253 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/a70deeb45a1344e1a51d9dec97580c06, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/81e99b7c9812431795b2b4c8d830d1dc, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/c091938486fe46529338d07f8e95834b] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp, totalSize=36.0 K 2024-11-21T21:16:25,253 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/b0d82f917cb045929a326078e6b44487, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/d4c4b16a680d4e9cac7d0cc51224cee9, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/8264820331154d49865b560580514a3f] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp, totalSize=38.2 K 2024-11-21T21:16:25,253 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting a70deeb45a1344e1a51d9dec97580c06, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732223781460 2024-11-21T21:16:25,253 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0d82f917cb045929a326078e6b44487, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732223781460 2024-11-21T21:16:25,253 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4c4b16a680d4e9cac7d0cc51224cee9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1732223782156 2024-11-21T21:16:25,253 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 81e99b7c9812431795b2b4c8d830d1dc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1732223782156 2024-11-21T21:16:25,253 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8264820331154d49865b560580514a3f, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732223783352 2024-11-21T21:16:25,253 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting c091938486fe46529338d07f8e95834b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732223783352 2024-11-21T21:16:25,258 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d4d04f4516ef90780bbe9af2779af72#A#compaction#434 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:16:25,258 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d4d04f4516ef90780bbe9af2779af72#B#compaction#435 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:16:25,259 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/bfde29406b6248cdba66776129c2abf4 is 50, key is test_row_0/A:col10/1732223784012/Put/seqid=0 2024-11-21T21:16:25,259 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/374f79dbb97044dd80eb029e697d8875 is 50, key is test_row_0/B:col10/1732223784012/Put/seqid=0 2024-11-21T21:16:25,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742328_1504 (size=12561) 2024-11-21T21:16:25,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742327_1503 (size=12663) 2024-11-21T21:16:25,267 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/bfde29406b6248cdba66776129c2abf4 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/bfde29406b6248cdba66776129c2abf4 2024-11-21T21:16:25,267 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/374f79dbb97044dd80eb029e697d8875 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/374f79dbb97044dd80eb029e697d8875 2024-11-21T21:16:25,273 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d4d04f4516ef90780bbe9af2779af72/A of 0d4d04f4516ef90780bbe9af2779af72 into bfde29406b6248cdba66776129c2abf4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:16:25,273 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d4d04f4516ef90780bbe9af2779af72/B of 0d4d04f4516ef90780bbe9af2779af72 into 374f79dbb97044dd80eb029e697d8875(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:16:25,273 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d4d04f4516ef90780bbe9af2779af72: 2024-11-21T21:16:25,273 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d4d04f4516ef90780bbe9af2779af72: 2024-11-21T21:16:25,273 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., storeName=0d4d04f4516ef90780bbe9af2779af72/A, priority=13, startTime=1732223785252; duration=0sec 2024-11-21T21:16:25,273 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., storeName=0d4d04f4516ef90780bbe9af2779af72/B, priority=13, startTime=1732223785252; duration=0sec 2024-11-21T21:16:25,274 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:16:25,274 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d4d04f4516ef90780bbe9af2779af72:A 2024-11-21T21:16:25,274 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:25,274 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d4d04f4516ef90780bbe9af2779af72:B 2024-11-21T21:16:25,274 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:16:25,274 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:16:25,274 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): 0d4d04f4516ef90780bbe9af2779af72/C is initiating minor compaction (all files) 2024-11-21T21:16:25,274 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d4d04f4516ef90780bbe9af2779af72/C in TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:25,274 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/b1c50f70f10741a0a1f13d57c8377002, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/43ce028d9c9d4411a00d9d4c7b54cbb1, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/cc99c46375ff4eafa776dbb8f6988d75] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp, totalSize=36.0 K 2024-11-21T21:16:25,274 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1c50f70f10741a0a1f13d57c8377002, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732223781460 2024-11-21T21:16:25,275 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43ce028d9c9d4411a00d9d4c7b54cbb1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1732223782156 2024-11-21T21:16:25,275 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc99c46375ff4eafa776dbb8f6988d75, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732223783352 2024-11-21T21:16:25,280 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d4d04f4516ef90780bbe9af2779af72#C#compaction#436 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:16:25,281 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/3060e0b3af734345a3b3c47c0da4a8e4 is 50, key is test_row_0/C:col10/1732223784012/Put/seqid=0 2024-11-21T21:16:25,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742329_1505 (size=12663) 2024-11-21T21:16:25,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-21T21:16:25,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:25,315 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 0d4d04f4516ef90780bbe9af2779af72 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-21T21:16:25,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=A 2024-11-21T21:16:25,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:25,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=B 2024-11-21T21:16:25,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:25,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=C 2024-11-21T21:16:25,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:25,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/26a1cab2f3cc4d75b8671162a4e970df is 50, key is test_row_0/A:col10/1732223784035/Put/seqid=0 2024-11-21T21:16:25,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742330_1506 (size=12151) 2024-11-21T21:16:25,693 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/3060e0b3af734345a3b3c47c0da4a8e4 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/3060e0b3af734345a3b3c47c0da4a8e4 2024-11-21T21:16:25,696 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d4d04f4516ef90780bbe9af2779af72/C of 0d4d04f4516ef90780bbe9af2779af72 into 3060e0b3af734345a3b3c47c0da4a8e4(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:16:25,697 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d4d04f4516ef90780bbe9af2779af72: 2024-11-21T21:16:25,697 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., storeName=0d4d04f4516ef90780bbe9af2779af72/C, priority=13, startTime=1732223785252; duration=0sec 2024-11-21T21:16:25,697 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:25,697 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d4d04f4516ef90780bbe9af2779af72:C 2024-11-21T21:16:25,723 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/26a1cab2f3cc4d75b8671162a4e970df 2024-11-21T21:16:25,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/9c9bed854abf424aa0f0e8b4f2bac0a3 is 50, key is test_row_0/B:col10/1732223784035/Put/seqid=0 2024-11-21T21:16:25,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742331_1507 (size=12151) 2024-11-21T21:16:26,132 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/9c9bed854abf424aa0f0e8b4f2bac0a3 2024-11-21T21:16:26,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/ed22fb6bd9524f24b8710f36ccb4ca6b is 50, key is test_row_0/C:col10/1732223784035/Put/seqid=0 2024-11-21T21:16:26,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742332_1508 (size=12151) 2024-11-21T21:16:26,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-21T21:16:26,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 0d4d04f4516ef90780bbe9af2779af72 2024-11-21T21:16:26,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. as already flushing 2024-11-21T21:16:26,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:26,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223846291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:26,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:26,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:26,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:26,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:26,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223846299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:26,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:26,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:26,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:26,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:26,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223846410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:26,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:26,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:26,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223846410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:26,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:26,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:26,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:26,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:26,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:26,546 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/ed22fb6bd9524f24b8710f36ccb4ca6b 2024-11-21T21:16:26,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/26a1cab2f3cc4d75b8671162a4e970df as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/26a1cab2f3cc4d75b8671162a4e970df 2024-11-21T21:16:26,552 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/26a1cab2f3cc4d75b8671162a4e970df, entries=150, sequenceid=221, filesize=11.9 K 2024-11-21T21:16:26,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/9c9bed854abf424aa0f0e8b4f2bac0a3 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/9c9bed854abf424aa0f0e8b4f2bac0a3 2024-11-21T21:16:26,556 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/9c9bed854abf424aa0f0e8b4f2bac0a3, entries=150, sequenceid=221, filesize=11.9 K 2024-11-21T21:16:26,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/ed22fb6bd9524f24b8710f36ccb4ca6b as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/ed22fb6bd9524f24b8710f36ccb4ca6b 2024-11-21T21:16:26,560 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/ed22fb6bd9524f24b8710f36ccb4ca6b, entries=150, sequenceid=221, filesize=11.9 K 2024-11-21T21:16:26,561 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 0d4d04f4516ef90780bbe9af2779af72 in 1245ms, sequenceid=221, compaction requested=false 2024-11-21T21:16:26,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 0d4d04f4516ef90780bbe9af2779af72: 2024-11-21T21:16:26,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:26,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-21T21:16:26,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-21T21:16:26,564 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-21T21:16:26,564 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4690 sec 2024-11-21T21:16:26,566 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 2.4740 sec 2024-11-21T21:16:26,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 0d4d04f4516ef90780bbe9af2779af72 2024-11-21T21:16:26,628 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0d4d04f4516ef90780bbe9af2779af72 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-21T21:16:26,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=A 2024-11-21T21:16:26,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:26,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=B 2024-11-21T21:16:26,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:26,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=C 2024-11-21T21:16:26,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:26,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/5707774bbc7546438772ce947537f32b is 50, key is test_row_0/A:col10/1732223786289/Put/seqid=0 2024-11-21T21:16:26,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742333_1509 (size=14541) 2024-11-21T21:16:26,636 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/5707774bbc7546438772ce947537f32b 2024-11-21T21:16:26,644 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/945581c7ed294645b117d24e4265344f is 50, key is test_row_0/B:col10/1732223786289/Put/seqid=0 2024-11-21T21:16:26,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742334_1510 (size=12151) 2024-11-21T21:16:26,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:26,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223846668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:26,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:26,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:26,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:26,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:26,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223846676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:26,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:26,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:26,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:26,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:26,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223846780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:26,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:26,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:26,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:26,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:26,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223846790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:26,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:26,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:26,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:26,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:26,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223846990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:26,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:26,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:26,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:27,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:27,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223847010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:27,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:27,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:27,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:27,062 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/945581c7ed294645b117d24e4265344f 2024-11-21T21:16:27,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/259afc6459904e199f6ca06664762807 is 50, key is test_row_0/C:col10/1732223786289/Put/seqid=0 2024-11-21T21:16:27,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742335_1511 (size=12151) 2024-11-21T21:16:27,079 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/259afc6459904e199f6ca06664762807 2024-11-21T21:16:27,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/5707774bbc7546438772ce947537f32b as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/5707774bbc7546438772ce947537f32b 2024-11-21T21:16:27,087 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/5707774bbc7546438772ce947537f32b, entries=200, sequenceid=240, filesize=14.2 K 2024-11-21T21:16:27,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/945581c7ed294645b117d24e4265344f as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/945581c7ed294645b117d24e4265344f 2024-11-21T21:16:27,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/945581c7ed294645b117d24e4265344f, entries=150, sequenceid=240, filesize=11.9 K 2024-11-21T21:16:27,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/259afc6459904e199f6ca06664762807 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/259afc6459904e199f6ca06664762807 2024-11-21T21:16:27,095 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/259afc6459904e199f6ca06664762807, entries=150, sequenceid=240, filesize=11.9 K 2024-11-21T21:16:27,098 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 0d4d04f4516ef90780bbe9af2779af72 in 471ms, sequenceid=240, compaction requested=true 2024-11-21T21:16:27,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0d4d04f4516ef90780bbe9af2779af72: 2024-11-21T21:16:27,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d4d04f4516ef90780bbe9af2779af72:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:16:27,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:27,099 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:16:27,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d4d04f4516ef90780bbe9af2779af72:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:16:27,099 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:16:27,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:27,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d4d04f4516ef90780bbe9af2779af72:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:16:27,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:16:27,102 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:16:27,102 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:16:27,102 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): 0d4d04f4516ef90780bbe9af2779af72/A is initiating minor compaction (all files) 2024-11-21T21:16:27,102 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): 0d4d04f4516ef90780bbe9af2779af72/B is initiating minor compaction (all files) 2024-11-21T21:16:27,102 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d4d04f4516ef90780bbe9af2779af72/B in TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:27,102 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d4d04f4516ef90780bbe9af2779af72/A in TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:27,102 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/bfde29406b6248cdba66776129c2abf4, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/26a1cab2f3cc4d75b8671162a4e970df, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/5707774bbc7546438772ce947537f32b] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp, totalSize=38.3 K 2024-11-21T21:16:27,102 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/374f79dbb97044dd80eb029e697d8875, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/9c9bed854abf424aa0f0e8b4f2bac0a3, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/945581c7ed294645b117d24e4265344f] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp, totalSize=36.1 K 2024-11-21T21:16:27,103 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 374f79dbb97044dd80eb029e697d8875, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732223783352 2024-11-21T21:16:27,103 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfde29406b6248cdba66776129c2abf4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732223783352 2024-11-21T21:16:27,103 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c9bed854abf424aa0f0e8b4f2bac0a3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732223784035 2024-11-21T21:16:27,103 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26a1cab2f3cc4d75b8671162a4e970df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732223784035 2024-11-21T21:16:27,103 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5707774bbc7546438772ce947537f32b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732223786282 2024-11-21T21:16:27,103 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 945581c7ed294645b117d24e4265344f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732223786282 2024-11-21T21:16:27,115 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d4d04f4516ef90780bbe9af2779af72#A#compaction#443 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:16:27,115 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d4d04f4516ef90780bbe9af2779af72#B#compaction#444 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:16:27,116 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/9c43e4de703c4020b8cb201ffe8aa294 is 50, key is test_row_0/B:col10/1732223786289/Put/seqid=0 2024-11-21T21:16:27,116 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/3063ee033fd54f109f6dcea60e88b1b7 is 50, key is test_row_0/A:col10/1732223786289/Put/seqid=0 2024-11-21T21:16:27,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742336_1512 (size=12663) 2024-11-21T21:16:27,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742337_1513 (size=12765) 2024-11-21T21:16:27,149 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/3063ee033fd54f109f6dcea60e88b1b7 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/3063ee033fd54f109f6dcea60e88b1b7 2024-11-21T21:16:27,152 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/9c43e4de703c4020b8cb201ffe8aa294 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/9c43e4de703c4020b8cb201ffe8aa294 2024-11-21T21:16:27,155 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d4d04f4516ef90780bbe9af2779af72/A of 0d4d04f4516ef90780bbe9af2779af72 into 3063ee033fd54f109f6dcea60e88b1b7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:16:27,155 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d4d04f4516ef90780bbe9af2779af72: 2024-11-21T21:16:27,155 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., storeName=0d4d04f4516ef90780bbe9af2779af72/A, priority=13, startTime=1732223787099; duration=0sec 2024-11-21T21:16:27,155 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:16:27,155 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d4d04f4516ef90780bbe9af2779af72:A 2024-11-21T21:16:27,155 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:16:27,159 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:16:27,160 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): 0d4d04f4516ef90780bbe9af2779af72/C is initiating minor compaction (all files) 2024-11-21T21:16:27,160 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d4d04f4516ef90780bbe9af2779af72/C in TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:27,160 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d4d04f4516ef90780bbe9af2779af72/B of 0d4d04f4516ef90780bbe9af2779af72 into 9c43e4de703c4020b8cb201ffe8aa294(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:16:27,160 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d4d04f4516ef90780bbe9af2779af72: 2024-11-21T21:16:27,160 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., storeName=0d4d04f4516ef90780bbe9af2779af72/B, priority=13, startTime=1732223787099; duration=0sec 2024-11-21T21:16:27,160 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/3060e0b3af734345a3b3c47c0da4a8e4, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/ed22fb6bd9524f24b8710f36ccb4ca6b, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/259afc6459904e199f6ca06664762807] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp, totalSize=36.1 K 2024-11-21T21:16:27,160 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:27,160 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d4d04f4516ef90780bbe9af2779af72:B 2024-11-21T21:16:27,160 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3060e0b3af734345a3b3c47c0da4a8e4, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732223783352 2024-11-21T21:16:27,160 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed22fb6bd9524f24b8710f36ccb4ca6b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732223784035 2024-11-21T21:16:27,160 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 259afc6459904e199f6ca06664762807, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732223786282 2024-11-21T21:16:27,166 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d4d04f4516ef90780bbe9af2779af72#C#compaction#445 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:16:27,166 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/8d8cbf9ed04e4e50b122cab87ad93276 is 50, key is test_row_0/C:col10/1732223786289/Put/seqid=0 2024-11-21T21:16:27,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742338_1514 (size=12765) 2024-11-21T21:16:27,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 0d4d04f4516ef90780bbe9af2779af72 2024-11-21T21:16:27,315 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0d4d04f4516ef90780bbe9af2779af72 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-21T21:16:27,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=A 2024-11-21T21:16:27,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:27,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=B 2024-11-21T21:16:27,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:27,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=C 2024-11-21T21:16:27,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:27,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/5de962bb66a54f60aa6317d58848d5ab is 50, key is test_row_0/A:col10/1732223786657/Put/seqid=0 2024-11-21T21:16:27,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742339_1515 (size=14591) 2024-11-21T21:16:27,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:27,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223847365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:27,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:27,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:27,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:27,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:27,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223847368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:27,377 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:27,377 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:27,377 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:27,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:27,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223847480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:27,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:27,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:27,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223847480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:27,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:27,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:27,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:27,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:27,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:27,585 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/8d8cbf9ed04e4e50b122cab87ad93276 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/8d8cbf9ed04e4e50b122cab87ad93276 2024-11-21T21:16:27,589 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d4d04f4516ef90780bbe9af2779af72/C of 0d4d04f4516ef90780bbe9af2779af72 into 8d8cbf9ed04e4e50b122cab87ad93276(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:16:27,589 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d4d04f4516ef90780bbe9af2779af72: 2024-11-21T21:16:27,589 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., storeName=0d4d04f4516ef90780bbe9af2779af72/C, priority=13, startTime=1732223787100; duration=0sec 2024-11-21T21:16:27,589 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:27,589 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d4d04f4516ef90780bbe9af2779af72:C 2024-11-21T21:16:27,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:27,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223847690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:27,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:27,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223847690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:27,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:27,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:27,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:27,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:27,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:27,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:27,722 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/5de962bb66a54f60aa6317d58848d5ab 2024-11-21T21:16:27,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/fc67162efbca41ebadef3d87c8bc097a is 50, key is test_row_0/B:col10/1732223786657/Put/seqid=0 2024-11-21T21:16:27,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742340_1516 (size=12201) 2024-11-21T21:16:28,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:28,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223848010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:28,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:28,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:28,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:28,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223848010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:28,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:28,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:28,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:28,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:28,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/fc67162efbca41ebadef3d87c8bc097a 2024-11-21T21:16:28,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/f3a8e1af29e347f1a9d179bea24e3698 is 50, key is test_row_0/C:col10/1732223786657/Put/seqid=0 2024-11-21T21:16:28,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742341_1517 (size=12201) 2024-11-21T21:16:28,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-21T21:16:28,261 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:16:28,261 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:16:28,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-21T21:16:28,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-21T21:16:28,263 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:16:28,263 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:16:28,263 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:16:28,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-21T21:16:28,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-21T21:16:28,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:28,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. as already flushing 2024-11-21T21:16:28,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:28,416 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:28,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:28,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:28,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:28,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223848521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:28,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:28,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:28,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:28,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:28,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223848521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:28,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:28,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:28,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:28,560 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/f3a8e1af29e347f1a9d179bea24e3698 2024-11-21T21:16:28,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/5de962bb66a54f60aa6317d58848d5ab as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/5de962bb66a54f60aa6317d58848d5ab 2024-11-21T21:16:28,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/5de962bb66a54f60aa6317d58848d5ab, entries=200, sequenceid=257, filesize=14.2 K 2024-11-21T21:16:28,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/fc67162efbca41ebadef3d87c8bc097a as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/fc67162efbca41ebadef3d87c8bc097a 2024-11-21T21:16:28,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-21T21:16:28,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:28,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. as already flushing 2024-11-21T21:16:28,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:28,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:28,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:28,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:28,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/fc67162efbca41ebadef3d87c8bc097a, entries=150, sequenceid=257, filesize=11.9 K 2024-11-21T21:16:28,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/f3a8e1af29e347f1a9d179bea24e3698 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/f3a8e1af29e347f1a9d179bea24e3698 2024-11-21T21:16:28,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/f3a8e1af29e347f1a9d179bea24e3698, entries=150, sequenceid=257, filesize=11.9 K 2024-11-21T21:16:28,574 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 0d4d04f4516ef90780bbe9af2779af72 in 1259ms, sequenceid=257, compaction requested=false 2024-11-21T21:16:28,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0d4d04f4516ef90780bbe9af2779af72: 2024-11-21T21:16:28,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-21T21:16:28,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-21T21:16:28,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:28,721 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 0d4d04f4516ef90780bbe9af2779af72 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-21T21:16:28,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=A 2024-11-21T21:16:28,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:28,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=B 2024-11-21T21:16:28,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:28,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=C 2024-11-21T21:16:28,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:28,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/77b409d994d64c949a41ffc31959b034 is 50, key is test_row_0/A:col10/1732223787367/Put/seqid=0 2024-11-21T21:16:28,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742342_1518 (size=12301) 2024-11-21T21:16:28,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-21T21:16:29,130 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/77b409d994d64c949a41ffc31959b034 2024-11-21T21:16:29,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/3b7b311c79034edea0da4f98d1b83751 is 50, key is test_row_0/B:col10/1732223787367/Put/seqid=0 2024-11-21T21:16:29,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742343_1519 (size=12301) 2024-11-21T21:16:29,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-21T21:16:29,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 0d4d04f4516ef90780bbe9af2779af72 2024-11-21T21:16:29,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. as already flushing 2024-11-21T21:16:29,542 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/3b7b311c79034edea0da4f98d1b83751 2024-11-21T21:16:29,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/b98f4883ed35467aaabeb5e61b87d2bb is 50, key is test_row_0/C:col10/1732223787367/Put/seqid=0 2024-11-21T21:16:29,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742344_1520 (size=12301) 2024-11-21T21:16:29,550 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/b98f4883ed35467aaabeb5e61b87d2bb 2024-11-21T21:16:29,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/77b409d994d64c949a41ffc31959b034 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/77b409d994d64c949a41ffc31959b034 2024-11-21T21:16:29,556 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/77b409d994d64c949a41ffc31959b034, entries=150, sequenceid=275, filesize=12.0 K 2024-11-21T21:16:29,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/3b7b311c79034edea0da4f98d1b83751 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/3b7b311c79034edea0da4f98d1b83751 2024-11-21T21:16:29,560 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/3b7b311c79034edea0da4f98d1b83751, entries=150, sequenceid=275, filesize=12.0 K 2024-11-21T21:16:29,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/b98f4883ed35467aaabeb5e61b87d2bb as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/b98f4883ed35467aaabeb5e61b87d2bb 2024-11-21T21:16:29,564 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/b98f4883ed35467aaabeb5e61b87d2bb, entries=150, sequenceid=275, filesize=12.0 K 2024-11-21T21:16:29,564 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=60.38 KB/61830 for 0d4d04f4516ef90780bbe9af2779af72 in 843ms, sequenceid=275, compaction requested=true 2024-11-21T21:16:29,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 0d4d04f4516ef90780bbe9af2779af72: 2024-11-21T21:16:29,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:29,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-21T21:16:29,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-21T21:16:29,567 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-21T21:16:29,567 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3020 sec 2024-11-21T21:16:29,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 0d4d04f4516ef90780bbe9af2779af72 2024-11-21T21:16:29,567 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0d4d04f4516ef90780bbe9af2779af72 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-21T21:16:29,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=A 2024-11-21T21:16:29,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:29,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=B 2024-11-21T21:16:29,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:29,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=C 2024-11-21T21:16:29,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:29,568 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 1.3060 sec 2024-11-21T21:16:29,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/1f172bdfd2634b06bd5f6b88d3e78f4d is 50, key is test_row_0/A:col10/1732223789566/Put/seqid=0 2024-11-21T21:16:29,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742345_1521 (size=17181) 2024-11-21T21:16:29,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:29,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223849626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:29,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:29,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:29,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:29,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:29,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223849627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:29,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:29,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:29,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:29,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:29,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223849730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:29,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:29,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223849730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:29,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:29,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:29,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:29,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:29,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:29,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:29,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:29,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223849951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:29,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:29,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:29,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223849951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:29,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:29,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:29,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:29,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:29,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:29,975 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/1f172bdfd2634b06bd5f6b88d3e78f4d 2024-11-21T21:16:29,981 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/5a877d4e4211478b8ea1526431d5189a is 50, key is test_row_0/B:col10/1732223789566/Put/seqid=0 2024-11-21T21:16:29,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742346_1522 (size=12301) 2024-11-21T21:16:30,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:30,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223850270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:30,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:30,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:30,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:30,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:30,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223850270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:30,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:30,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:30,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:30,384 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/5a877d4e4211478b8ea1526431d5189a 2024-11-21T21:16:30,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/bb2ae3785ce142579930e7aa7e008ef2 is 50, key is test_row_0/C:col10/1732223789566/Put/seqid=0 2024-11-21T21:16:30,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742347_1523 (size=12301) 2024-11-21T21:16:30,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-21T21:16:30,410 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:16:30,411 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:16:30,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-21T21:16:30,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-21T21:16:30,412 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:16:30,413 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:16:30,413 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:16:30,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-21T21:16:30,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-21T21:16:30,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:30,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. as already flushing 2024-11-21T21:16:30,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:30,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:30,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:30,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:30,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-21T21:16:30,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:30,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. as already flushing 2024-11-21T21:16:30,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:30,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:30,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:30,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:30,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-21T21:16:30,793 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/bb2ae3785ce142579930e7aa7e008ef2 2024-11-21T21:16:30,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/1f172bdfd2634b06bd5f6b88d3e78f4d as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/1f172bdfd2634b06bd5f6b88d3e78f4d 2024-11-21T21:16:30,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/1f172bdfd2634b06bd5f6b88d3e78f4d, entries=250, sequenceid=288, filesize=16.8 K 2024-11-21T21:16:30,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:30,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223850790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:30,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/5a877d4e4211478b8ea1526431d5189a as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/5a877d4e4211478b8ea1526431d5189a 2024-11-21T21:16:30,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:30,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:30,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223850790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:30,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:30,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:30,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:30,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:30,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:30,803 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/5a877d4e4211478b8ea1526431d5189a, entries=150, sequenceid=288, filesize=12.0 K 2024-11-21T21:16:30,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/bb2ae3785ce142579930e7aa7e008ef2 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/bb2ae3785ce142579930e7aa7e008ef2 2024-11-21T21:16:30,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/bb2ae3785ce142579930e7aa7e008ef2, entries=150, sequenceid=288, filesize=12.0 K 2024-11-21T21:16:30,811 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 0d4d04f4516ef90780bbe9af2779af72 in 1244ms, sequenceid=288, compaction requested=true 2024-11-21T21:16:30,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0d4d04f4516ef90780bbe9af2779af72: 2024-11-21T21:16:30,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d4d04f4516ef90780bbe9af2779af72:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:16:30,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:30,811 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:16:30,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d4d04f4516ef90780bbe9af2779af72:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:16:30,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:30,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d4d04f4516ef90780bbe9af2779af72:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:16:30,812 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:16:30,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:16:30,812 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56736 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:16:30,812 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49568 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:16:30,813 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): 0d4d04f4516ef90780bbe9af2779af72/B is initiating minor compaction (all files) 2024-11-21T21:16:30,813 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): 0d4d04f4516ef90780bbe9af2779af72/A is initiating minor compaction (all files) 2024-11-21T21:16:30,813 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d4d04f4516ef90780bbe9af2779af72/B in TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:30,813 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d4d04f4516ef90780bbe9af2779af72/A in TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:30,813 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/3063ee033fd54f109f6dcea60e88b1b7, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/5de962bb66a54f60aa6317d58848d5ab, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/77b409d994d64c949a41ffc31959b034, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/1f172bdfd2634b06bd5f6b88d3e78f4d] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp, totalSize=55.4 K 2024-11-21T21:16:30,813 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/9c43e4de703c4020b8cb201ffe8aa294, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/fc67162efbca41ebadef3d87c8bc097a, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/3b7b311c79034edea0da4f98d1b83751, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/5a877d4e4211478b8ea1526431d5189a] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp, totalSize=48.4 K 2024-11-21T21:16:30,813 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3063ee033fd54f109f6dcea60e88b1b7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732223786282 2024-11-21T21:16:30,813 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c43e4de703c4020b8cb201ffe8aa294, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732223786282 2024-11-21T21:16:30,813 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting fc67162efbca41ebadef3d87c8bc097a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732223786657 2024-11-21T21:16:30,813 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5de962bb66a54f60aa6317d58848d5ab, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732223786657 2024-11-21T21:16:30,813 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b7b311c79034edea0da4f98d1b83751, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732223787323 2024-11-21T21:16:30,814 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77b409d994d64c949a41ffc31959b034, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732223787323 2024-11-21T21:16:30,814 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f172bdfd2634b06bd5f6b88d3e78f4d, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732223789546 2024-11-21T21:16:30,814 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a877d4e4211478b8ea1526431d5189a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732223789546 2024-11-21T21:16:30,822 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d4d04f4516ef90780bbe9af2779af72#B#compaction#456 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:16:30,822 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d4d04f4516ef90780bbe9af2779af72#A#compaction#455 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:16:30,822 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/7af5c84c8d55495d87851f9228089f36 is 50, key is test_row_0/A:col10/1732223789566/Put/seqid=0 2024-11-21T21:16:30,822 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/01e35c4cc9274970a63a02aeb16e7f86 is 50, key is test_row_0/B:col10/1732223789566/Put/seqid=0 2024-11-21T21:16:30,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742348_1524 (size=13051) 2024-11-21T21:16:30,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742349_1525 (size=12949) 2024-11-21T21:16:30,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-21T21:16:30,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:30,870 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing 0d4d04f4516ef90780bbe9af2779af72 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-21T21:16:30,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=A 2024-11-21T21:16:30,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:30,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=B 2024-11-21T21:16:30,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:30,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=C 2024-11-21T21:16:30,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:30,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/41c5ae84867f413b911b7e4c989b5ac1 is 50, key is test_row_0/A:col10/1732223789611/Put/seqid=0 2024-11-21T21:16:30,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742350_1526 (size=12301) 2024-11-21T21:16:31,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-21T21:16:31,229 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/01e35c4cc9274970a63a02aeb16e7f86 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/01e35c4cc9274970a63a02aeb16e7f86 2024-11-21T21:16:31,229 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/7af5c84c8d55495d87851f9228089f36 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/7af5c84c8d55495d87851f9228089f36 2024-11-21T21:16:31,232 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0d4d04f4516ef90780bbe9af2779af72/B of 0d4d04f4516ef90780bbe9af2779af72 into 01e35c4cc9274970a63a02aeb16e7f86(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:16:31,232 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0d4d04f4516ef90780bbe9af2779af72/A of 0d4d04f4516ef90780bbe9af2779af72 into 7af5c84c8d55495d87851f9228089f36(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:16:31,232 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d4d04f4516ef90780bbe9af2779af72: 2024-11-21T21:16:31,232 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d4d04f4516ef90780bbe9af2779af72: 2024-11-21T21:16:31,232 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., storeName=0d4d04f4516ef90780bbe9af2779af72/B, priority=12, startTime=1732223790811; duration=0sec 2024-11-21T21:16:31,232 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., storeName=0d4d04f4516ef90780bbe9af2779af72/A, priority=12, startTime=1732223790811; duration=0sec 2024-11-21T21:16:31,233 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:16:31,233 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d4d04f4516ef90780bbe9af2779af72:A 2024-11-21T21:16:31,233 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:31,233 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d4d04f4516ef90780bbe9af2779af72:B 2024-11-21T21:16:31,233 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:16:31,233 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49568 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:16:31,233 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): 0d4d04f4516ef90780bbe9af2779af72/C is initiating minor compaction (all files) 2024-11-21T21:16:31,234 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d4d04f4516ef90780bbe9af2779af72/C in TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:31,234 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/8d8cbf9ed04e4e50b122cab87ad93276, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/f3a8e1af29e347f1a9d179bea24e3698, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/b98f4883ed35467aaabeb5e61b87d2bb, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/bb2ae3785ce142579930e7aa7e008ef2] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp, totalSize=48.4 K 2024-11-21T21:16:31,234 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d8cbf9ed04e4e50b122cab87ad93276, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732223786282 2024-11-21T21:16:31,234 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3a8e1af29e347f1a9d179bea24e3698, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732223786657 2024-11-21T21:16:31,234 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting b98f4883ed35467aaabeb5e61b87d2bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732223787323 2024-11-21T21:16:31,234 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb2ae3785ce142579930e7aa7e008ef2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732223789546 2024-11-21T21:16:31,241 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d4d04f4516ef90780bbe9af2779af72#C#compaction#458 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:16:31,241 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/7d3ab8f79a244952889d8ccb3936d7e4 is 50, key is test_row_0/C:col10/1732223789566/Put/seqid=0 2024-11-21T21:16:31,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742351_1527 (size=13051) 2024-11-21T21:16:31,278 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/41c5ae84867f413b911b7e4c989b5ac1 2024-11-21T21:16:31,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/5b4fc7497f1141b69380688ad7db396a is 50, key is test_row_0/B:col10/1732223789611/Put/seqid=0 2024-11-21T21:16:31,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742352_1528 (size=12301) 2024-11-21T21:16:31,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-21T21:16:31,648 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/7d3ab8f79a244952889d8ccb3936d7e4 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/7d3ab8f79a244952889d8ccb3936d7e4 2024-11-21T21:16:31,651 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0d4d04f4516ef90780bbe9af2779af72/C of 0d4d04f4516ef90780bbe9af2779af72 into 7d3ab8f79a244952889d8ccb3936d7e4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:16:31,651 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d4d04f4516ef90780bbe9af2779af72: 2024-11-21T21:16:31,651 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., storeName=0d4d04f4516ef90780bbe9af2779af72/C, priority=12, startTime=1732223790812; duration=0sec 2024-11-21T21:16:31,651 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:31,651 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d4d04f4516ef90780bbe9af2779af72:C 2024-11-21T21:16:31,687 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/5b4fc7497f1141b69380688ad7db396a 2024-11-21T21:16:31,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/879eae201b014644994c998aad04b14f is 50, key is test_row_0/C:col10/1732223789611/Put/seqid=0 2024-11-21T21:16:31,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742353_1529 (size=12301) 2024-11-21T21:16:31,696 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/879eae201b014644994c998aad04b14f 2024-11-21T21:16:31,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/41c5ae84867f413b911b7e4c989b5ac1 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/41c5ae84867f413b911b7e4c989b5ac1 2024-11-21T21:16:31,702 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/41c5ae84867f413b911b7e4c989b5ac1, entries=150, sequenceid=307, filesize=12.0 K 2024-11-21T21:16:31,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/5b4fc7497f1141b69380688ad7db396a as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/5b4fc7497f1141b69380688ad7db396a 2024-11-21T21:16:31,705 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/5b4fc7497f1141b69380688ad7db396a, entries=150, sequenceid=307, filesize=12.0 K 2024-11-21T21:16:31,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/879eae201b014644994c998aad04b14f as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/879eae201b014644994c998aad04b14f 2024-11-21T21:16:31,709 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/879eae201b014644994c998aad04b14f, entries=150, sequenceid=307, filesize=12.0 K 2024-11-21T21:16:31,709 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for 0d4d04f4516ef90780bbe9af2779af72 in 839ms, sequenceid=307, compaction requested=false 2024-11-21T21:16:31,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for 0d4d04f4516ef90780bbe9af2779af72: 2024-11-21T21:16:31,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:31,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-21T21:16:31,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-21T21:16:31,712 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-21T21:16:31,712 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2970 sec 2024-11-21T21:16:31,713 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 1.3010 sec 2024-11-21T21:16:31,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 0d4d04f4516ef90780bbe9af2779af72 2024-11-21T21:16:31,827 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0d4d04f4516ef90780bbe9af2779af72 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T21:16:31,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=A 2024-11-21T21:16:31,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:31,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=B 2024-11-21T21:16:31,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:31,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=C 2024-11-21T21:16:31,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:31,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/2481e64c79d44c139a28cec2277b6da4 is 50, key is test_row_0/A:col10/1732223791825/Put/seqid=0 2024-11-21T21:16:31,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742354_1530 (size=19621) 2024-11-21T21:16:31,838 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/2481e64c79d44c139a28cec2277b6da4 2024-11-21T21:16:31,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/4a9f3bbfcbd448408bf5b507b8375295 is 50, key is test_row_0/B:col10/1732223791825/Put/seqid=0 2024-11-21T21:16:31,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742355_1531 (size=12301) 2024-11-21T21:16:31,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732223791868, max timestamp=1732223791889Num uniques 859; 2024-11-21T21:16:31,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732223791868, max timestamp=1732223791889Num uniques 859; 2024-11-21T21:16:31,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T21:16:31,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T21:16:31,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732223791868, max timestamp=1732223791889Num uniques 859; 2024-11-21T21:16:31,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T21:16:31,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:31,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223851956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:31,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:31,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:31,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:31,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:31,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223851966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:31,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:31,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:31,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:32,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:32,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223852070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:32,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:32,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:32,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:32,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:32,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223852080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:32,093 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:32,093 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:32,093 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:32,246 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/4a9f3bbfcbd448408bf5b507b8375295 2024-11-21T21:16:32,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/674fd97bbfe54362b261a06325040414 is 50, key is test_row_0/C:col10/1732223791825/Put/seqid=0 2024-11-21T21:16:32,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742356_1532 (size=12301) 2024-11-21T21:16:32,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:32,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223852290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:32,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:32,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:32,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:32,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:32,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223852300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:32,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:32,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:32,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:32,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-21T21:16:32,561 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:16:32,561 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:16:32,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees 2024-11-21T21:16:32,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-21T21:16:32,562 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:16:32,563 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:16:32,563 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:16:32,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:32,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223852610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:32,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:32,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:32,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:32,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:32,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223852620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:32,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:32,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:32,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:32,654 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/674fd97bbfe54362b261a06325040414 2024-11-21T21:16:32,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/2481e64c79d44c139a28cec2277b6da4 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/2481e64c79d44c139a28cec2277b6da4 2024-11-21T21:16:32,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/2481e64c79d44c139a28cec2277b6da4, entries=300, sequenceid=320, filesize=19.2 K 2024-11-21T21:16:32,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/4a9f3bbfcbd448408bf5b507b8375295 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/4a9f3bbfcbd448408bf5b507b8375295 2024-11-21T21:16:32,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/4a9f3bbfcbd448408bf5b507b8375295, entries=150, sequenceid=320, filesize=12.0 K 2024-11-21T21:16:32,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/674fd97bbfe54362b261a06325040414 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/674fd97bbfe54362b261a06325040414 2024-11-21T21:16:32,666 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/674fd97bbfe54362b261a06325040414, entries=150, sequenceid=320, filesize=12.0 K 2024-11-21T21:16:32,667 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 0d4d04f4516ef90780bbe9af2779af72 in 840ms, sequenceid=320, compaction requested=true 2024-11-21T21:16:32,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0d4d04f4516ef90780bbe9af2779af72: 2024-11-21T21:16:32,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d4d04f4516ef90780bbe9af2779af72:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:16:32,667 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:16:32,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:32,667 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:16:32,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d4d04f4516ef90780bbe9af2779af72:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:16:32,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:32,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d4d04f4516ef90780bbe9af2779af72:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:16:32,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:16:32,668 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 44871 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:16:32,668 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:16:32,668 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): 0d4d04f4516ef90780bbe9af2779af72/A is initiating minor compaction (all files) 2024-11-21T21:16:32,668 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): 0d4d04f4516ef90780bbe9af2779af72/B is initiating minor compaction (all files) 2024-11-21T21:16:32,668 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d4d04f4516ef90780bbe9af2779af72/A in TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:32,668 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d4d04f4516ef90780bbe9af2779af72/B in TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:32,668 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/7af5c84c8d55495d87851f9228089f36, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/41c5ae84867f413b911b7e4c989b5ac1, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/2481e64c79d44c139a28cec2277b6da4] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp, totalSize=43.8 K 2024-11-21T21:16:32,668 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/01e35c4cc9274970a63a02aeb16e7f86, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/5b4fc7497f1141b69380688ad7db396a, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/4a9f3bbfcbd448408bf5b507b8375295] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp, totalSize=36.8 K 2024-11-21T21:16:32,668 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7af5c84c8d55495d87851f9228089f36, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732223789546 2024-11-21T21:16:32,668 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 01e35c4cc9274970a63a02aeb16e7f86, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732223789546 2024-11-21T21:16:32,669 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b4fc7497f1141b69380688ad7db396a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732223789611 2024-11-21T21:16:32,669 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41c5ae84867f413b911b7e4c989b5ac1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732223789611 2024-11-21T21:16:32,669 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a9f3bbfcbd448408bf5b507b8375295, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732223791823 2024-11-21T21:16:32,669 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2481e64c79d44c139a28cec2277b6da4, keycount=300, bloomtype=ROW, size=19.2 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732223791811 2024-11-21T21:16:32,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-21T21:16:32,677 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d4d04f4516ef90780bbe9af2779af72#A#compaction#464 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:16:32,677 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/612c5b0d09a7424d8537acc68c013a61 is 50, key is test_row_0/A:col10/1732223791825/Put/seqid=0 2024-11-21T21:16:32,679 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d4d04f4516ef90780bbe9af2779af72#B#compaction#465 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:16:32,679 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/aa48b5472b704354b66448275db693f2 is 50, key is test_row_0/B:col10/1732223791825/Put/seqid=0 2024-11-21T21:16:32,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742357_1533 (size=13051) 2024-11-21T21:16:32,684 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/612c5b0d09a7424d8537acc68c013a61 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/612c5b0d09a7424d8537acc68c013a61 2024-11-21T21:16:32,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742358_1534 (size=13153) 2024-11-21T21:16:32,688 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d4d04f4516ef90780bbe9af2779af72/A of 0d4d04f4516ef90780bbe9af2779af72 into 612c5b0d09a7424d8537acc68c013a61(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:16:32,688 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d4d04f4516ef90780bbe9af2779af72: 2024-11-21T21:16:32,688 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., storeName=0d4d04f4516ef90780bbe9af2779af72/A, priority=13, startTime=1732223792667; duration=0sec 2024-11-21T21:16:32,688 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:16:32,688 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d4d04f4516ef90780bbe9af2779af72:A 2024-11-21T21:16:32,688 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:16:32,689 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:16:32,689 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): 0d4d04f4516ef90780bbe9af2779af72/C is initiating minor compaction (all files) 2024-11-21T21:16:32,689 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d4d04f4516ef90780bbe9af2779af72/C in TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:32,689 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/7d3ab8f79a244952889d8ccb3936d7e4, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/879eae201b014644994c998aad04b14f, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/674fd97bbfe54362b261a06325040414] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp, totalSize=36.8 K 2024-11-21T21:16:32,690 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d3ab8f79a244952889d8ccb3936d7e4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732223789546 2024-11-21T21:16:32,690 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 879eae201b014644994c998aad04b14f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732223789611 2024-11-21T21:16:32,690 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 674fd97bbfe54362b261a06325040414, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732223791823 2024-11-21T21:16:32,696 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d4d04f4516ef90780bbe9af2779af72#C#compaction#466 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:16:32,696 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/5e09b271431a4cbd824b040485d2a90b is 50, key is test_row_0/C:col10/1732223791825/Put/seqid=0 2024-11-21T21:16:32,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742359_1535 (size=13153) 2024-11-21T21:16:32,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-21T21:16:32,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:32,715 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2902): Flushing 0d4d04f4516ef90780bbe9af2779af72 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-21T21:16:32,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=A 2024-11-21T21:16:32,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:16:32,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=B 2024-11-21T21:16:32,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:16:32,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=C 2024-11-21T21:16:32,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:16:32,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/344a05578da644fe925e119e23df5d96 is 50, key is test_row_0/A:col10/1732223791943/Put/seqid=0 2024-11-21T21:16:32,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742360_1536 (size=12301) 2024-11-21T21:16:32,725 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/344a05578da644fe925e119e23df5d96 2024-11-21T21:16:32,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/fa636c164e9142f18a51a53805ad9f80 is 50, key is test_row_0/B:col10/1732223791943/Put/seqid=0 2024-11-21T21:16:32,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742361_1537 (size=12301) 2024-11-21T21:16:32,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-21T21:16:33,089 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/aa48b5472b704354b66448275db693f2 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/aa48b5472b704354b66448275db693f2 2024-11-21T21:16:33,092 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d4d04f4516ef90780bbe9af2779af72/B of 0d4d04f4516ef90780bbe9af2779af72 into aa48b5472b704354b66448275db693f2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:16:33,092 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d4d04f4516ef90780bbe9af2779af72: 2024-11-21T21:16:33,092 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., storeName=0d4d04f4516ef90780bbe9af2779af72/B, priority=13, startTime=1732223792667; duration=0sec 2024-11-21T21:16:33,092 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:33,092 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d4d04f4516ef90780bbe9af2779af72:B 2024-11-21T21:16:33,107 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/5e09b271431a4cbd824b040485d2a90b as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/5e09b271431a4cbd824b040485d2a90b 2024-11-21T21:16:33,110 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d4d04f4516ef90780bbe9af2779af72/C of 0d4d04f4516ef90780bbe9af2779af72 into 5e09b271431a4cbd824b040485d2a90b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:16:33,110 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d4d04f4516ef90780bbe9af2779af72: 2024-11-21T21:16:33,110 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., storeName=0d4d04f4516ef90780bbe9af2779af72/C, priority=13, startTime=1732223792668; duration=0sec 2024-11-21T21:16:33,110 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:33,110 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d4d04f4516ef90780bbe9af2779af72:C 2024-11-21T21:16:33,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 0d4d04f4516ef90780bbe9af2779af72 2024-11-21T21:16:33,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. as already flushing 2024-11-21T21:16:33,135 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/fa636c164e9142f18a51a53805ad9f80 2024-11-21T21:16:33,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/c87f562c77464b3cbc02b172dfcd54be is 50, key is test_row_0/C:col10/1732223791943/Put/seqid=0 2024-11-21T21:16:33,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742362_1538 (size=12301) 2024-11-21T21:16:33,144 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/c87f562c77464b3cbc02b172dfcd54be 2024-11-21T21:16:33,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/344a05578da644fe925e119e23df5d96 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/344a05578da644fe925e119e23df5d96 2024-11-21T21:16:33,150 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/344a05578da644fe925e119e23df5d96, entries=150, sequenceid=357, filesize=12.0 K 2024-11-21T21:16:33,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/fa636c164e9142f18a51a53805ad9f80 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/fa636c164e9142f18a51a53805ad9f80 2024-11-21T21:16:33,153 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/fa636c164e9142f18a51a53805ad9f80, entries=150, sequenceid=357, filesize=12.0 K 2024-11-21T21:16:33,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/c87f562c77464b3cbc02b172dfcd54be as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/c87f562c77464b3cbc02b172dfcd54be 2024-11-21T21:16:33,157 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/c87f562c77464b3cbc02b172dfcd54be, entries=150, sequenceid=357, filesize=12.0 K 2024-11-21T21:16:33,157 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=33.54 KB/34350 for 0d4d04f4516ef90780bbe9af2779af72 in 442ms, sequenceid=357, compaction requested=false 2024-11-21T21:16:33,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2603): Flush status journal for 0d4d04f4516ef90780bbe9af2779af72: 2024-11-21T21:16:33,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:33,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142 2024-11-21T21:16:33,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=142 2024-11-21T21:16:33,160 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-21T21:16:33,160 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 596 msec 2024-11-21T21:16:33,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 0d4d04f4516ef90780bbe9af2779af72 2024-11-21T21:16:33,160 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0d4d04f4516ef90780bbe9af2779af72 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T21:16:33,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=A 2024-11-21T21:16:33,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:33,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=B 2024-11-21T21:16:33,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:33,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=C 2024-11-21T21:16:33,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:33,162 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees in 600 msec 2024-11-21T21:16:33,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/01f6e1462b49497e9b47bd3652665f94 is 50, key is test_row_0/A:col10/1732223793158/Put/seqid=0 2024-11-21T21:16:33,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742363_1539 (size=17181) 2024-11-21T21:16:33,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-21T21:16:33,190 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:16:33,190 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:16:33,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees 2024-11-21T21:16:33,191 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:16:33,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-21T21:16:33,192 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:16:33,192 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:16:33,233 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732223793198, max timestamp=1732223793215Num uniques 859; 2024-11-21T21:16:33,233 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T21:16:33,236 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732223793198, max timestamp=1732223793215Num uniques 859; 2024-11-21T21:16:33,236 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T21:16:33,236 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732223793198, max timestamp=1732223793215Num uniques 859; 2024-11-21T21:16:33,236 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-21T21:16:33,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-21T21:16:33,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:33,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223853330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:33,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:33,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:33,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:33,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-21T21:16:33,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:33,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. as already flushing 2024-11-21T21:16:33,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:33,344 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:33,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:33,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:33,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:33,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223853332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:33,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:33,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:33,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:33,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:33,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223853440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:33,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:33,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:33,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:33,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:33,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223853450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:33,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:33,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:33,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:33,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-21T21:16:33,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:33,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. as already flushing 2024-11-21T21:16:33,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:33,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:33,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:33,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:33,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-21T21:16:33,574 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/01f6e1462b49497e9b47bd3652665f94 2024-11-21T21:16:33,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/ffa4d8448af04e32878a6302fad2364f is 50, key is test_row_0/B:col10/1732223793158/Put/seqid=0 2024-11-21T21:16:33,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742364_1540 (size=12301) 2024-11-21T21:16:33,583 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/ffa4d8448af04e32878a6302fad2364f 2024-11-21T21:16:33,587 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/8b957c9424674be9adec52c9b1f27bf2 is 50, key is test_row_0/C:col10/1732223793158/Put/seqid=0 2024-11-21T21:16:33,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742365_1541 (size=12301) 2024-11-21T21:16:33,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-21T21:16:33,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:33,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. as already flushing 2024-11-21T21:16:33,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:33,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:33,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:33,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:33,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:33,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223853660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:33,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:33,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:33,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:33,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:33,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223853660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:33,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:33,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:33,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:33,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-21T21:16:33,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:33,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. as already flushing 2024-11-21T21:16:33,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:33,802 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:33,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:33,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:33,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-21T21:16:33,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-21T21:16:33,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:33,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. as already flushing 2024-11-21T21:16:33,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:33,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:33,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:33,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:33,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:33,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54370 deadline: 1732223853980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:33,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:33,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:33,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54376 deadline: 1732223853980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:33,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:33,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:33,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 , the old value is region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:33,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:33,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., hostname=849dee24a242,41213,1732223653469, seqNum=2 because the exception is null or not the one we care about 2024-11-21T21:16:33,998 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/8b957c9424674be9adec52c9b1f27bf2 2024-11-21T21:16:34,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/01f6e1462b49497e9b47bd3652665f94 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/01f6e1462b49497e9b47bd3652665f94 2024-11-21T21:16:34,003 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/01f6e1462b49497e9b47bd3652665f94, entries=250, sequenceid=369, filesize=16.8 K 2024-11-21T21:16:34,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/ffa4d8448af04e32878a6302fad2364f as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/ffa4d8448af04e32878a6302fad2364f 2024-11-21T21:16:34,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/ffa4d8448af04e32878a6302fad2364f, entries=150, sequenceid=369, filesize=12.0 K 2024-11-21T21:16:34,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/8b957c9424674be9adec52c9b1f27bf2 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/8b957c9424674be9adec52c9b1f27bf2 2024-11-21T21:16:34,009 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/8b957c9424674be9adec52c9b1f27bf2, entries=150, sequenceid=369, filesize=12.0 K 2024-11-21T21:16:34,009 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 0d4d04f4516ef90780bbe9af2779af72 in 849ms, sequenceid=369, compaction requested=true 2024-11-21T21:16:34,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0d4d04f4516ef90780bbe9af2779af72: 2024-11-21T21:16:34,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d4d04f4516ef90780bbe9af2779af72:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:16:34,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:34,010 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:16:34,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d4d04f4516ef90780bbe9af2779af72:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:16:34,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:34,010 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:16:34,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d4d04f4516ef90780bbe9af2779af72:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:16:34,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:16:34,010 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:16:34,010 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42533 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:16:34,010 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): 0d4d04f4516ef90780bbe9af2779af72/A is initiating minor compaction (all files) 2024-11-21T21:16:34,010 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): 0d4d04f4516ef90780bbe9af2779af72/B is initiating minor compaction (all files) 2024-11-21T21:16:34,011 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d4d04f4516ef90780bbe9af2779af72/B in TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:34,011 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d4d04f4516ef90780bbe9af2779af72/A in TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:34,011 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/612c5b0d09a7424d8537acc68c013a61, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/344a05578da644fe925e119e23df5d96, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/01f6e1462b49497e9b47bd3652665f94] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp, totalSize=41.5 K 2024-11-21T21:16:34,011 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/aa48b5472b704354b66448275db693f2, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/fa636c164e9142f18a51a53805ad9f80, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/ffa4d8448af04e32878a6302fad2364f] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp, totalSize=36.9 K 2024-11-21T21:16:34,011 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa48b5472b704354b66448275db693f2, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732223791823 2024-11-21T21:16:34,011 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 612c5b0d09a7424d8537acc68c013a61, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732223791823 2024-11-21T21:16:34,011 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 344a05578da644fe925e119e23df5d96, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732223791943 2024-11-21T21:16:34,012 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa636c164e9142f18a51a53805ad9f80, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732223791943 2024-11-21T21:16:34,012 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 01f6e1462b49497e9b47bd3652665f94, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732223793144 2024-11-21T21:16:34,012 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffa4d8448af04e32878a6302fad2364f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732223793154 2024-11-21T21:16:34,017 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d4d04f4516ef90780bbe9af2779af72#B#compaction#473 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:16:34,017 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/64d9bf3b3a3d41729e21cb35c65baea6 is 50, key is test_row_0/B:col10/1732223793158/Put/seqid=0 2024-11-21T21:16:34,020 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d4d04f4516ef90780bbe9af2779af72#A#compaction#474 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:16:34,020 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/183b4ebc3acd41089744a471075429c9 is 50, key is test_row_0/A:col10/1732223793158/Put/seqid=0 2024-11-21T21:16:34,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742366_1542 (size=13255) 2024-11-21T21:16:34,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742367_1543 (size=13153) 2024-11-21T21:16:34,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-21T21:16:34,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:34,107 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2902): Flushing 0d4d04f4516ef90780bbe9af2779af72 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-21T21:16:34,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=A 2024-11-21T21:16:34,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:16:34,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=B 2024-11-21T21:16:34,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:16:34,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=C 2024-11-21T21:16:34,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:16:34,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/a6821d5abfa941a1bf6e2275506269bc is 50, key is test_row_0/A:col10/1732223793317/Put/seqid=0 2024-11-21T21:16:34,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742368_1544 (size=12301) 2024-11-21T21:16:34,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-21T21:16:34,357 INFO [Thread-2063 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2063. 2024-11-21T21:16:34,357 DEBUG [Thread-2063 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:16:34,357 DEBUG [Thread-2063 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:34,357 DEBUG [Thread-2063 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:34,357 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:16:34,363 INFO [Thread-2059 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2059. 2024-11-21T21:16:34,363 DEBUG [Thread-2059 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:16:34,363 DEBUG [Thread-2059 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:34,363 DEBUG [Thread-2059 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:34,363 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:16:34,364 INFO [Thread-2061 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2061. 2024-11-21T21:16:34,364 DEBUG [Thread-2061 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:16:34,364 DEBUG [Thread-2061 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:34,364 DEBUG [Thread-2061 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:34,364 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:16:34,367 INFO [Thread-2057 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2057. 2024-11-21T21:16:34,367 DEBUG [Thread-2057 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:16:34,367 DEBUG [Thread-2057 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:34,367 DEBUG [Thread-2057 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:34,367 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:16:34,373 INFO [Thread-2055 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2055. 2024-11-21T21:16:34,373 DEBUG [Thread-2055 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:16:34,373 DEBUG [Thread-2055 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:34,373 DEBUG [Thread-2055 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:34,373 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:16:34,437 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/64d9bf3b3a3d41729e21cb35c65baea6 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/64d9bf3b3a3d41729e21cb35c65baea6 2024-11-21T21:16:34,437 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/183b4ebc3acd41089744a471075429c9 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/183b4ebc3acd41089744a471075429c9 2024-11-21T21:16:34,443 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d4d04f4516ef90780bbe9af2779af72/B of 0d4d04f4516ef90780bbe9af2779af72 into 64d9bf3b3a3d41729e21cb35c65baea6(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:16:34,443 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d4d04f4516ef90780bbe9af2779af72: 2024-11-21T21:16:34,443 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., storeName=0d4d04f4516ef90780bbe9af2779af72/B, priority=13, startTime=1732223794010; duration=0sec 2024-11-21T21:16:34,443 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:16:34,443 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d4d04f4516ef90780bbe9af2779af72:B 2024-11-21T21:16:34,443 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d4d04f4516ef90780bbe9af2779af72/A of 0d4d04f4516ef90780bbe9af2779af72 into 183b4ebc3acd41089744a471075429c9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:16:34,443 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d4d04f4516ef90780bbe9af2779af72: 2024-11-21T21:16:34,443 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:16:34,443 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., storeName=0d4d04f4516ef90780bbe9af2779af72/A, priority=13, startTime=1732223794010; duration=0sec 2024-11-21T21:16:34,443 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:34,443 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d4d04f4516ef90780bbe9af2779af72:A 2024-11-21T21:16:34,444 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:16:34,444 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): 0d4d04f4516ef90780bbe9af2779af72/C is initiating minor compaction (all files) 2024-11-21T21:16:34,444 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d4d04f4516ef90780bbe9af2779af72/C in TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:34,445 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/5e09b271431a4cbd824b040485d2a90b, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/c87f562c77464b3cbc02b172dfcd54be, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/8b957c9424674be9adec52c9b1f27bf2] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp, totalSize=36.9 K 2024-11-21T21:16:34,445 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e09b271431a4cbd824b040485d2a90b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732223791823 2024-11-21T21:16:34,445 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting c87f562c77464b3cbc02b172dfcd54be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732223791943 2024-11-21T21:16:34,446 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b957c9424674be9adec52c9b1f27bf2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732223793154 2024-11-21T21:16:34,451 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d4d04f4516ef90780bbe9af2779af72#C#compaction#476 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:16:34,451 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/5e8437273a134bf7bdc09779c7699e59 is 50, key is test_row_0/C:col10/1732223793158/Put/seqid=0 2024-11-21T21:16:34,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742369_1545 (size=13255) 2024-11-21T21:16:34,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 0d4d04f4516ef90780bbe9af2779af72 2024-11-21T21:16:34,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. as already flushing 2024-11-21T21:16:34,493 INFO [Thread-2046 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2046. 2024-11-21T21:16:34,493 INFO [Thread-2044 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2044. 2024-11-21T21:16:34,493 DEBUG [Thread-2046 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:16:34,493 DEBUG [Thread-2044 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:16:34,493 DEBUG [Thread-2046 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:34,493 DEBUG [Thread-2044 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:34,493 DEBUG [Thread-2046 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:34,493 DEBUG [Thread-2044 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:34,493 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:16:34,493 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:16:34,517 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/a6821d5abfa941a1bf6e2275506269bc 2024-11-21T21:16:34,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/eea28ac898ca4f54bedc6cf1654b4c2b is 50, key is test_row_0/B:col10/1732223793317/Put/seqid=0 2024-11-21T21:16:34,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742370_1546 (size=12301) 2024-11-21T21:16:34,862 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/5e8437273a134bf7bdc09779c7699e59 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/5e8437273a134bf7bdc09779c7699e59 2024-11-21T21:16:34,869 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d4d04f4516ef90780bbe9af2779af72/C of 0d4d04f4516ef90780bbe9af2779af72 into 5e8437273a134bf7bdc09779c7699e59(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:16:34,869 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d4d04f4516ef90780bbe9af2779af72: 2024-11-21T21:16:34,869 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., storeName=0d4d04f4516ef90780bbe9af2779af72/C, priority=13, startTime=1732223794010; duration=0sec 2024-11-21T21:16:34,869 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:34,869 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d4d04f4516ef90780bbe9af2779af72:C 2024-11-21T21:16:34,933 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/eea28ac898ca4f54bedc6cf1654b4c2b 2024-11-21T21:16:34,936 INFO [Thread-2050 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2050. 2024-11-21T21:16:34,936 DEBUG [Thread-2050 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:16:34,936 DEBUG [Thread-2050 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:34,936 DEBUG [Thread-2050 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:34,936 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:16:34,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/edf6a87a6a1f4abaa30b984ccedf8d70 is 50, key is test_row_0/C:col10/1732223793317/Put/seqid=0 2024-11-21T21:16:34,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742371_1547 (size=12301) 2024-11-21T21:16:35,004 INFO [Thread-2048 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2048. 2024-11-21T21:16:35,004 DEBUG [Thread-2048 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:16:35,004 DEBUG [Thread-2048 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:35,004 DEBUG [Thread-2048 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:35,004 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:16:35,045 INFO [Thread-2052 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2052. 2024-11-21T21:16:35,045 DEBUG [Thread-2052 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:16:35,045 DEBUG [Thread-2052 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:35,045 DEBUG [Thread-2052 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:35,046 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:16:35,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-21T21:16:35,348 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/edf6a87a6a1f4abaa30b984ccedf8d70 2024-11-21T21:16:35,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/a6821d5abfa941a1bf6e2275506269bc as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/a6821d5abfa941a1bf6e2275506269bc 2024-11-21T21:16:35,360 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/a6821d5abfa941a1bf6e2275506269bc, entries=150, sequenceid=406, filesize=12.0 K 2024-11-21T21:16:35,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/eea28ac898ca4f54bedc6cf1654b4c2b as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/eea28ac898ca4f54bedc6cf1654b4c2b 2024-11-21T21:16:35,365 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/eea28ac898ca4f54bedc6cf1654b4c2b, entries=150, sequenceid=406, filesize=12.0 K 2024-11-21T21:16:35,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/edf6a87a6a1f4abaa30b984ccedf8d70 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/edf6a87a6a1f4abaa30b984ccedf8d70 2024-11-21T21:16:35,370 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/edf6a87a6a1f4abaa30b984ccedf8d70, entries=150, sequenceid=406, filesize=12.0 K 2024-11-21T21:16:35,371 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=33.54 KB/34350 for 0d4d04f4516ef90780bbe9af2779af72 in 1264ms, sequenceid=406, compaction requested=false 2024-11-21T21:16:35,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2603): Flush status journal for 0d4d04f4516ef90780bbe9af2779af72: 2024-11-21T21:16:35,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:35,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=144 2024-11-21T21:16:35,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=144 2024-11-21T21:16:35,375 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-21T21:16:35,375 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1810 sec 2024-11-21T21:16:35,377 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees in 2.1860 sec 2024-11-21T21:16:37,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-21T21:16:37,352 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:16:37,352 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T21:16:37,353 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 141 2024-11-21T21:16:37,353 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 139 2024-11-21T21:16:37,353 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-11-21T21:16:37,353 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 10 2024-11-21T21:16:37,353 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-11-21T21:16:37,353 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T21:16:37,353 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T21:16:37,353 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1207 2024-11-21T21:16:37,353 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3621 rows 2024-11-21T21:16:37,353 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1202 2024-11-21T21:16:37,353 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3606 rows 2024-11-21T21:16:37,353 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1206 2024-11-21T21:16:37,353 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3610 rows 2024-11-21T21:16:37,353 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1202 2024-11-21T21:16:37,353 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3602 rows 2024-11-21T21:16:37,353 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1204 2024-11-21T21:16:37,353 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3609 rows 2024-11-21T21:16:37,354 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T21:16:37,354 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T21:16:37,354 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:37,354 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:37,355 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:16:37,356 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-21T21:16:37,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T21:16:37,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-21T21:16:37,361 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732223797361"}]},"ts":"1732223797361"} 2024-11-21T21:16:37,363 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T21:16:37,363 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T21:16:37,363 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T21:16:37,365 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d4d04f4516ef90780bbe9af2779af72, UNASSIGN}] 2024-11-21T21:16:37,366 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d4d04f4516ef90780bbe9af2779af72, UNASSIGN 2024-11-21T21:16:37,366 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=0d4d04f4516ef90780bbe9af2779af72, regionState=CLOSING, regionLocation=849dee24a242,41213,1732223653469 2024-11-21T21:16:37,368 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d4d04f4516ef90780bbe9af2779af72, UNASSIGN because future has completed 2024-11-21T21:16:37,368 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T21:16:37,368 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469}] 2024-11-21T21:16:37,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-21T21:16:37,521 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(122): Close 0d4d04f4516ef90780bbe9af2779af72 2024-11-21T21:16:37,521 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T21:16:37,521 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1722): Closing 0d4d04f4516ef90780bbe9af2779af72, disabling compactions & flushes 2024-11-21T21:16:37,521 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:37,521 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:37,521 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. after waiting 0 ms 2024-11-21T21:16:37,521 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:37,522 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(2902): Flushing 0d4d04f4516ef90780bbe9af2779af72 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-21T21:16:37,522 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=A 2024-11-21T21:16:37,522 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:37,522 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=B 2024-11-21T21:16:37,522 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:37,522 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0d4d04f4516ef90780bbe9af2779af72, store=C 2024-11-21T21:16:37,523 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:37,530 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/ddfb7be9efc84f4b830ddd64b290eadd is 50, key is test_row_0/A:col10/1732223795042/Put/seqid=0 2024-11-21T21:16:37,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742372_1548 (size=12301) 2024-11-21T21:16:37,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-21T21:16:37,936 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/ddfb7be9efc84f4b830ddd64b290eadd 2024-11-21T21:16:37,947 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/5b48e18c89e94dc685aab9c87a348036 is 50, key is test_row_0/B:col10/1732223795042/Put/seqid=0 2024-11-21T21:16:37,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742373_1549 (size=12301) 2024-11-21T21:16:37,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-21T21:16:38,352 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/5b48e18c89e94dc685aab9c87a348036 2024-11-21T21:16:38,359 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/ce54c8801a0f4b18ae232a7b8fa5277b is 50, key is test_row_0/C:col10/1732223795042/Put/seqid=0 2024-11-21T21:16:38,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742374_1550 (size=12301) 2024-11-21T21:16:38,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-21T21:16:38,763 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/ce54c8801a0f4b18ae232a7b8fa5277b 2024-11-21T21:16:38,767 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/A/ddfb7be9efc84f4b830ddd64b290eadd as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/ddfb7be9efc84f4b830ddd64b290eadd 2024-11-21T21:16:38,770 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/ddfb7be9efc84f4b830ddd64b290eadd, entries=150, sequenceid=417, filesize=12.0 K 2024-11-21T21:16:38,771 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/B/5b48e18c89e94dc685aab9c87a348036 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/5b48e18c89e94dc685aab9c87a348036 2024-11-21T21:16:38,774 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/5b48e18c89e94dc685aab9c87a348036, entries=150, sequenceid=417, filesize=12.0 K 2024-11-21T21:16:38,775 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/.tmp/C/ce54c8801a0f4b18ae232a7b8fa5277b as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/ce54c8801a0f4b18ae232a7b8fa5277b 2024-11-21T21:16:38,778 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/ce54c8801a0f4b18ae232a7b8fa5277b, entries=150, sequenceid=417, filesize=12.0 K 2024-11-21T21:16:38,779 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 0d4d04f4516ef90780bbe9af2779af72 in 1257ms, sequenceid=417, compaction requested=true 2024-11-21T21:16:38,780 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/55cbccfcf4c6471cae4fd674a10f7c97, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/0bdce1b80d444cbab3a8406ab2f06eee, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/826a6cf5619442138b37ae34fc32afc8, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/73299cc37b0148cbbfaaa7c82171496f, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/d3fab06d0387441bb3faf2c9313f280c, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/1c1917909d104f9f95ca0c2f0273bb5e, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/b2cf8c0e0e634ad7919cfb611ca633be, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/c8e6d48a61644d699a0879585eaf7183, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/137e20e38b2145c8bb5a4776332f6a7d, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/b0d82f917cb045929a326078e6b44487, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/d4c4b16a680d4e9cac7d0cc51224cee9, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/8264820331154d49865b560580514a3f, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/bfde29406b6248cdba66776129c2abf4, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/26a1cab2f3cc4d75b8671162a4e970df, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/5707774bbc7546438772ce947537f32b, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/3063ee033fd54f109f6dcea60e88b1b7, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/5de962bb66a54f60aa6317d58848d5ab, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/77b409d994d64c949a41ffc31959b034, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/1f172bdfd2634b06bd5f6b88d3e78f4d, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/7af5c84c8d55495d87851f9228089f36, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/41c5ae84867f413b911b7e4c989b5ac1, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/2481e64c79d44c139a28cec2277b6da4, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/612c5b0d09a7424d8537acc68c013a61, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/344a05578da644fe925e119e23df5d96, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/01f6e1462b49497e9b47bd3652665f94] to archive 2024-11-21T21:16:38,780 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T21:16:38,782 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/55cbccfcf4c6471cae4fd674a10f7c97 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/55cbccfcf4c6471cae4fd674a10f7c97 2024-11-21T21:16:38,783 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/0bdce1b80d444cbab3a8406ab2f06eee to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/0bdce1b80d444cbab3a8406ab2f06eee 2024-11-21T21:16:38,784 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/826a6cf5619442138b37ae34fc32afc8 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/826a6cf5619442138b37ae34fc32afc8 2024-11-21T21:16:38,785 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/73299cc37b0148cbbfaaa7c82171496f to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/73299cc37b0148cbbfaaa7c82171496f 2024-11-21T21:16:38,786 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/d3fab06d0387441bb3faf2c9313f280c to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/d3fab06d0387441bb3faf2c9313f280c 2024-11-21T21:16:38,787 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/1c1917909d104f9f95ca0c2f0273bb5e to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/1c1917909d104f9f95ca0c2f0273bb5e 2024-11-21T21:16:38,788 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/b2cf8c0e0e634ad7919cfb611ca633be to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/b2cf8c0e0e634ad7919cfb611ca633be 2024-11-21T21:16:38,788 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/c8e6d48a61644d699a0879585eaf7183 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/c8e6d48a61644d699a0879585eaf7183 2024-11-21T21:16:38,789 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/137e20e38b2145c8bb5a4776332f6a7d to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/137e20e38b2145c8bb5a4776332f6a7d 2024-11-21T21:16:38,790 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/b0d82f917cb045929a326078e6b44487 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/b0d82f917cb045929a326078e6b44487 2024-11-21T21:16:38,791 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/d4c4b16a680d4e9cac7d0cc51224cee9 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/d4c4b16a680d4e9cac7d0cc51224cee9 2024-11-21T21:16:38,791 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/8264820331154d49865b560580514a3f to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/8264820331154d49865b560580514a3f 2024-11-21T21:16:38,792 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/bfde29406b6248cdba66776129c2abf4 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/bfde29406b6248cdba66776129c2abf4 2024-11-21T21:16:38,793 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/26a1cab2f3cc4d75b8671162a4e970df to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/26a1cab2f3cc4d75b8671162a4e970df 2024-11-21T21:16:38,794 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/5707774bbc7546438772ce947537f32b to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/5707774bbc7546438772ce947537f32b 2024-11-21T21:16:38,794 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/3063ee033fd54f109f6dcea60e88b1b7 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/3063ee033fd54f109f6dcea60e88b1b7 2024-11-21T21:16:38,795 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/5de962bb66a54f60aa6317d58848d5ab to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/5de962bb66a54f60aa6317d58848d5ab 2024-11-21T21:16:38,796 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/77b409d994d64c949a41ffc31959b034 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/77b409d994d64c949a41ffc31959b034 2024-11-21T21:16:38,797 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/1f172bdfd2634b06bd5f6b88d3e78f4d to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/1f172bdfd2634b06bd5f6b88d3e78f4d 2024-11-21T21:16:38,797 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/7af5c84c8d55495d87851f9228089f36 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/7af5c84c8d55495d87851f9228089f36 2024-11-21T21:16:38,798 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/41c5ae84867f413b911b7e4c989b5ac1 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/41c5ae84867f413b911b7e4c989b5ac1 2024-11-21T21:16:38,799 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/2481e64c79d44c139a28cec2277b6da4 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/2481e64c79d44c139a28cec2277b6da4 2024-11-21T21:16:38,799 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/612c5b0d09a7424d8537acc68c013a61 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/612c5b0d09a7424d8537acc68c013a61 2024-11-21T21:16:38,800 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/344a05578da644fe925e119e23df5d96 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/344a05578da644fe925e119e23df5d96 2024-11-21T21:16:38,801 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/01f6e1462b49497e9b47bd3652665f94 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/01f6e1462b49497e9b47bd3652665f94 2024-11-21T21:16:38,802 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/4c304c5baf2e452da39ca41f6b08663f, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/6a2081e3f1964249a3f2b3f9e9c837cc, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/fbc3a0b1aa014936afe2f9c0d931a1e4, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/c4f1003113a640f485b006a189aee416, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/7854b366381a40fb99148e65458ca0ac, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/7adcbca81c334d8a904c8732ba128e5e, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/c01b9c94d0054e7e925efecec308fa5e, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/3d75aa6f58f34874a6e7f55a96fa5725, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/4f339f2b38c041a991ac56ad31c68906, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/42578f688e8243b5a5b6fefbf2631e8d, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/25bc371dd7744957a9d751affa84d049, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/a70deeb45a1344e1a51d9dec97580c06, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/7d64fd8d3a034b3f8c35fa28f63f9bae, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/81e99b7c9812431795b2b4c8d830d1dc, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/374f79dbb97044dd80eb029e697d8875, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/c091938486fe46529338d07f8e95834b, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/9c9bed854abf424aa0f0e8b4f2bac0a3, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/9c43e4de703c4020b8cb201ffe8aa294, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/945581c7ed294645b117d24e4265344f, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/fc67162efbca41ebadef3d87c8bc097a, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/3b7b311c79034edea0da4f98d1b83751, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/01e35c4cc9274970a63a02aeb16e7f86, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/5a877d4e4211478b8ea1526431d5189a, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/5b4fc7497f1141b69380688ad7db396a, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/aa48b5472b704354b66448275db693f2, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/4a9f3bbfcbd448408bf5b507b8375295, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/fa636c164e9142f18a51a53805ad9f80, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/ffa4d8448af04e32878a6302fad2364f] to archive 2024-11-21T21:16:38,803 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T21:16:38,803 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/4c304c5baf2e452da39ca41f6b08663f to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/4c304c5baf2e452da39ca41f6b08663f 2024-11-21T21:16:38,804 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/6a2081e3f1964249a3f2b3f9e9c837cc to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/6a2081e3f1964249a3f2b3f9e9c837cc 2024-11-21T21:16:38,805 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/fbc3a0b1aa014936afe2f9c0d931a1e4 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/fbc3a0b1aa014936afe2f9c0d931a1e4 2024-11-21T21:16:38,805 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/c4f1003113a640f485b006a189aee416 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/c4f1003113a640f485b006a189aee416 2024-11-21T21:16:38,806 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/7854b366381a40fb99148e65458ca0ac to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/7854b366381a40fb99148e65458ca0ac 2024-11-21T21:16:38,807 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/7adcbca81c334d8a904c8732ba128e5e to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/7adcbca81c334d8a904c8732ba128e5e 2024-11-21T21:16:38,807 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/c01b9c94d0054e7e925efecec308fa5e to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/c01b9c94d0054e7e925efecec308fa5e 2024-11-21T21:16:38,808 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/3d75aa6f58f34874a6e7f55a96fa5725 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/3d75aa6f58f34874a6e7f55a96fa5725 2024-11-21T21:16:38,809 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/4f339f2b38c041a991ac56ad31c68906 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/4f339f2b38c041a991ac56ad31c68906 2024-11-21T21:16:38,809 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/42578f688e8243b5a5b6fefbf2631e8d to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/42578f688e8243b5a5b6fefbf2631e8d 2024-11-21T21:16:38,810 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/25bc371dd7744957a9d751affa84d049 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/25bc371dd7744957a9d751affa84d049 2024-11-21T21:16:38,811 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/a70deeb45a1344e1a51d9dec97580c06 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/a70deeb45a1344e1a51d9dec97580c06 2024-11-21T21:16:38,811 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/7d64fd8d3a034b3f8c35fa28f63f9bae to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/7d64fd8d3a034b3f8c35fa28f63f9bae 2024-11-21T21:16:38,812 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/81e99b7c9812431795b2b4c8d830d1dc to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/81e99b7c9812431795b2b4c8d830d1dc 2024-11-21T21:16:38,813 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/374f79dbb97044dd80eb029e697d8875 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/374f79dbb97044dd80eb029e697d8875 2024-11-21T21:16:38,813 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/c091938486fe46529338d07f8e95834b to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/c091938486fe46529338d07f8e95834b 2024-11-21T21:16:38,814 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/9c9bed854abf424aa0f0e8b4f2bac0a3 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/9c9bed854abf424aa0f0e8b4f2bac0a3 2024-11-21T21:16:38,815 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/9c43e4de703c4020b8cb201ffe8aa294 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/9c43e4de703c4020b8cb201ffe8aa294 2024-11-21T21:16:38,815 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/945581c7ed294645b117d24e4265344f to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/945581c7ed294645b117d24e4265344f 2024-11-21T21:16:38,816 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/fc67162efbca41ebadef3d87c8bc097a to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/fc67162efbca41ebadef3d87c8bc097a 2024-11-21T21:16:38,817 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/3b7b311c79034edea0da4f98d1b83751 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/3b7b311c79034edea0da4f98d1b83751 2024-11-21T21:16:38,817 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/01e35c4cc9274970a63a02aeb16e7f86 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/01e35c4cc9274970a63a02aeb16e7f86 2024-11-21T21:16:38,818 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/5a877d4e4211478b8ea1526431d5189a to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/5a877d4e4211478b8ea1526431d5189a 2024-11-21T21:16:38,819 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/5b4fc7497f1141b69380688ad7db396a to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/5b4fc7497f1141b69380688ad7db396a 2024-11-21T21:16:38,819 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/aa48b5472b704354b66448275db693f2 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/aa48b5472b704354b66448275db693f2 2024-11-21T21:16:38,820 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/4a9f3bbfcbd448408bf5b507b8375295 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/4a9f3bbfcbd448408bf5b507b8375295 2024-11-21T21:16:38,821 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/fa636c164e9142f18a51a53805ad9f80 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/fa636c164e9142f18a51a53805ad9f80 2024-11-21T21:16:38,821 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/ffa4d8448af04e32878a6302fad2364f to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/ffa4d8448af04e32878a6302fad2364f 2024-11-21T21:16:38,823 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/326106c26c9147c892b68c6f94151041, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/7c9ecb8e6e5d4055b8bd67ec5d3ec0de, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/446cb42ef52849cdbfd90f766ec33a8a, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/5170de82639741bcb7db9dbb513b588b, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/3814b52c1def4d98bcd31fa196ff7462, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/6c9e9c739e3f4414a37ba278f8b232d2, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/63a2059163314a5fa3a9dec82c953771, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/62312e0341554caeadbee904c6033a34, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/f37352248ecc4dc0bc8ee343bd2e6433, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/d30f8cfb83a94694a62ed01368d35ee5, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/9ffd660cebf4444d94917cd364800075, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/b1c50f70f10741a0a1f13d57c8377002, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/541c30a7334f4729ac235868b1ca5f0f, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/43ce028d9c9d4411a00d9d4c7b54cbb1, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/3060e0b3af734345a3b3c47c0da4a8e4, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/cc99c46375ff4eafa776dbb8f6988d75, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/ed22fb6bd9524f24b8710f36ccb4ca6b, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/8d8cbf9ed04e4e50b122cab87ad93276, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/259afc6459904e199f6ca06664762807, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/f3a8e1af29e347f1a9d179bea24e3698, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/b98f4883ed35467aaabeb5e61b87d2bb, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/7d3ab8f79a244952889d8ccb3936d7e4, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/bb2ae3785ce142579930e7aa7e008ef2, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/879eae201b014644994c998aad04b14f, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/5e09b271431a4cbd824b040485d2a90b, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/674fd97bbfe54362b261a06325040414, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/c87f562c77464b3cbc02b172dfcd54be, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/8b957c9424674be9adec52c9b1f27bf2] to archive 2024-11-21T21:16:38,823 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T21:16:38,824 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/326106c26c9147c892b68c6f94151041 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/326106c26c9147c892b68c6f94151041 2024-11-21T21:16:38,825 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/7c9ecb8e6e5d4055b8bd67ec5d3ec0de to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/7c9ecb8e6e5d4055b8bd67ec5d3ec0de 2024-11-21T21:16:38,826 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/446cb42ef52849cdbfd90f766ec33a8a to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/446cb42ef52849cdbfd90f766ec33a8a 2024-11-21T21:16:38,826 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/5170de82639741bcb7db9dbb513b588b to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/5170de82639741bcb7db9dbb513b588b 2024-11-21T21:16:38,827 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/3814b52c1def4d98bcd31fa196ff7462 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/3814b52c1def4d98bcd31fa196ff7462 2024-11-21T21:16:38,828 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/6c9e9c739e3f4414a37ba278f8b232d2 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/6c9e9c739e3f4414a37ba278f8b232d2 2024-11-21T21:16:38,828 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/63a2059163314a5fa3a9dec82c953771 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/63a2059163314a5fa3a9dec82c953771 2024-11-21T21:16:38,829 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/62312e0341554caeadbee904c6033a34 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/62312e0341554caeadbee904c6033a34 2024-11-21T21:16:38,830 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/f37352248ecc4dc0bc8ee343bd2e6433 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/f37352248ecc4dc0bc8ee343bd2e6433 2024-11-21T21:16:38,830 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/d30f8cfb83a94694a62ed01368d35ee5 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/d30f8cfb83a94694a62ed01368d35ee5 2024-11-21T21:16:38,831 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/9ffd660cebf4444d94917cd364800075 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/9ffd660cebf4444d94917cd364800075 2024-11-21T21:16:38,831 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/b1c50f70f10741a0a1f13d57c8377002 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/b1c50f70f10741a0a1f13d57c8377002 2024-11-21T21:16:38,832 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/541c30a7334f4729ac235868b1ca5f0f to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/541c30a7334f4729ac235868b1ca5f0f 2024-11-21T21:16:38,833 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/43ce028d9c9d4411a00d9d4c7b54cbb1 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/43ce028d9c9d4411a00d9d4c7b54cbb1 2024-11-21T21:16:38,833 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/3060e0b3af734345a3b3c47c0da4a8e4 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/3060e0b3af734345a3b3c47c0da4a8e4 2024-11-21T21:16:38,834 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/cc99c46375ff4eafa776dbb8f6988d75 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/cc99c46375ff4eafa776dbb8f6988d75 2024-11-21T21:16:38,834 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/ed22fb6bd9524f24b8710f36ccb4ca6b to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/ed22fb6bd9524f24b8710f36ccb4ca6b 2024-11-21T21:16:38,835 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/8d8cbf9ed04e4e50b122cab87ad93276 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/8d8cbf9ed04e4e50b122cab87ad93276 2024-11-21T21:16:38,836 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/259afc6459904e199f6ca06664762807 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/259afc6459904e199f6ca06664762807 2024-11-21T21:16:38,837 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/f3a8e1af29e347f1a9d179bea24e3698 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/f3a8e1af29e347f1a9d179bea24e3698 2024-11-21T21:16:38,837 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/b98f4883ed35467aaabeb5e61b87d2bb to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/b98f4883ed35467aaabeb5e61b87d2bb 2024-11-21T21:16:38,838 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/7d3ab8f79a244952889d8ccb3936d7e4 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/7d3ab8f79a244952889d8ccb3936d7e4 2024-11-21T21:16:38,839 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/bb2ae3785ce142579930e7aa7e008ef2 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/bb2ae3785ce142579930e7aa7e008ef2 2024-11-21T21:16:38,840 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/879eae201b014644994c998aad04b14f to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/879eae201b014644994c998aad04b14f 2024-11-21T21:16:38,841 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/5e09b271431a4cbd824b040485d2a90b to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/5e09b271431a4cbd824b040485d2a90b 2024-11-21T21:16:38,841 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/674fd97bbfe54362b261a06325040414 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/674fd97bbfe54362b261a06325040414 2024-11-21T21:16:38,842 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/c87f562c77464b3cbc02b172dfcd54be to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/c87f562c77464b3cbc02b172dfcd54be 2024-11-21T21:16:38,843 DEBUG [StoreCloser-TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/8b957c9424674be9adec52c9b1f27bf2 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/8b957c9424674be9adec52c9b1f27bf2 2024-11-21T21:16:38,846 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/recovered.edits/420.seqid, newMaxSeqId=420, maxSeqId=1 2024-11-21T21:16:38,846 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. 2024-11-21T21:16:38,847 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1676): Region close journal for 0d4d04f4516ef90780bbe9af2779af72: Waiting for close lock at 1732223797521Running coprocessor pre-close hooks at 1732223797521Disabling compacts and flushes for region at 1732223797521Disabling writes for close at 1732223797521Obtaining lock to block concurrent updates at 1732223797522 (+1 ms)Preparing flush snapshotting stores in 0d4d04f4516ef90780bbe9af2779af72 at 1732223797522Finished memstore snapshotting TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732223797523 (+1 ms)Flushing stores of TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72. at 1732223797524 (+1 ms)Flushing 0d4d04f4516ef90780bbe9af2779af72/A: creating writer at 1732223797524Flushing 0d4d04f4516ef90780bbe9af2779af72/A: appending metadata at 1732223797529 (+5 ms)Flushing 0d4d04f4516ef90780bbe9af2779af72/A: closing flushed file at 1732223797529Flushing 0d4d04f4516ef90780bbe9af2779af72/B: creating writer at 1732223797943 (+414 ms)Flushing 0d4d04f4516ef90780bbe9af2779af72/B: appending metadata at 1732223797947 (+4 ms)Flushing 0d4d04f4516ef90780bbe9af2779af72/B: closing flushed file at 1732223797947Flushing 0d4d04f4516ef90780bbe9af2779af72/C: creating writer at 1732223798356 (+409 ms)Flushing 0d4d04f4516ef90780bbe9af2779af72/C: appending metadata at 1732223798359 (+3 ms)Flushing 0d4d04f4516ef90780bbe9af2779af72/C: closing flushed file at 1732223798359Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60c7b28e: reopening flushed file at 1732223798766 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e8d7e66: reopening flushed file at 1732223798770 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29394ff0: reopening flushed file at 1732223798775 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 0d4d04f4516ef90780bbe9af2779af72 in 1257ms, sequenceid=417, compaction requested=true at 1732223798779 (+4 ms)Writing region close event to WAL at 1732223798844 (+65 ms)Running coprocessor post-close hooks at 1732223798846 (+2 ms)Closed at 1732223798846 2024-11-21T21:16:38,848 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(157): Closed 0d4d04f4516ef90780bbe9af2779af72 2024-11-21T21:16:38,849 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=0d4d04f4516ef90780bbe9af2779af72, regionState=CLOSED 2024-11-21T21:16:38,850 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 because future has completed 2024-11-21T21:16:38,851 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-21T21:16:38,851 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseRegionProcedure 0d4d04f4516ef90780bbe9af2779af72, server=849dee24a242,41213,1732223653469 in 1.4820 sec 2024-11-21T21:16:38,853 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-21T21:16:38,853 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d4d04f4516ef90780bbe9af2779af72, UNASSIGN in 1.4860 sec 2024-11-21T21:16:38,854 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-21T21:16:38,854 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4900 sec 2024-11-21T21:16:38,855 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732223798855"}]},"ts":"1732223798855"} 2024-11-21T21:16:38,856 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T21:16:38,856 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T21:16:38,858 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5010 sec 2024-11-21T21:16:39,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-21T21:16:39,512 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:16:39,513 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-21T21:16:39,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=149, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:16:39,518 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=149, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:16:39,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-21T21:16:39,520 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=149, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:16:39,524 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72 2024-11-21T21:16:39,528 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A, FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B, FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C, FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/recovered.edits] 2024-11-21T21:16:39,532 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/183b4ebc3acd41089744a471075429c9 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/183b4ebc3acd41089744a471075429c9 2024-11-21T21:16:39,534 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/a6821d5abfa941a1bf6e2275506269bc to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/a6821d5abfa941a1bf6e2275506269bc 2024-11-21T21:16:39,536 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/ddfb7be9efc84f4b830ddd64b290eadd to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/A/ddfb7be9efc84f4b830ddd64b290eadd 2024-11-21T21:16:39,539 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/5b48e18c89e94dc685aab9c87a348036 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/5b48e18c89e94dc685aab9c87a348036 2024-11-21T21:16:39,540 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/64d9bf3b3a3d41729e21cb35c65baea6 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/64d9bf3b3a3d41729e21cb35c65baea6 2024-11-21T21:16:39,542 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/eea28ac898ca4f54bedc6cf1654b4c2b to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/B/eea28ac898ca4f54bedc6cf1654b4c2b 2024-11-21T21:16:39,544 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/5e8437273a134bf7bdc09779c7699e59 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/5e8437273a134bf7bdc09779c7699e59 2024-11-21T21:16:39,546 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/ce54c8801a0f4b18ae232a7b8fa5277b to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/ce54c8801a0f4b18ae232a7b8fa5277b 2024-11-21T21:16:39,547 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/edf6a87a6a1f4abaa30b984ccedf8d70 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/C/edf6a87a6a1f4abaa30b984ccedf8d70 2024-11-21T21:16:39,550 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/recovered.edits/420.seqid to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72/recovered.edits/420.seqid 2024-11-21T21:16:39,551 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/0d4d04f4516ef90780bbe9af2779af72 2024-11-21T21:16:39,551 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T21:16:39,554 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=149, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:16:39,556 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T21:16:39,559 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T21:16:39,560 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=149, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:16:39,560 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T21:16:39,560 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732223799560"}]},"ts":"9223372036854775807"} 2024-11-21T21:16:39,562 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T21:16:39,562 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 0d4d04f4516ef90780bbe9af2779af72, NAME => 'TestAcidGuarantees,,1732223772136.0d4d04f4516ef90780bbe9af2779af72.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T21:16:39,563 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T21:16:39,563 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732223799563"}]},"ts":"9223372036854775807"} 2024-11-21T21:16:39,564 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T21:16:39,565 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=149, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:16:39,566 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 52 msec 2024-11-21T21:16:39,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-21T21:16:39,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T21:16:39,632 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:16:39,647 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250 (was 251), OpenFileDescriptor=447 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=403 (was 398) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=11429 (was 11467) 2024-11-21T21:16:39,656 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=403, ProcessCount=11, AvailableMemoryMB=11428 2024-11-21T21:16:39,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T21:16:39,658 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T21:16:39,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T21:16:39,660 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T21:16:39,660 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:16:39,660 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 150 2024-11-21T21:16:39,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-21T21:16:39,661 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T21:16:39,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742375_1551 (size=1061) 2024-11-21T21:16:39,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-21T21:16:39,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-21T21:16:40,073 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f 2024-11-21T21:16:40,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742376_1552 (size=53) 2024-11-21T21:16:40,081 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:16:40,081 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 5a7f333184706b95cbb8c49fb3d84f31, disabling compactions & flushes 2024-11-21T21:16:40,081 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:40,081 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:40,081 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. after waiting 0 ms 2024-11-21T21:16:40,081 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:40,081 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:40,081 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5a7f333184706b95cbb8c49fb3d84f31: Waiting for close lock at 1732223800081Disabling compacts and flushes for region at 1732223800081Disabling writes for close at 1732223800081Writing region close event to WAL at 1732223800081Closed at 1732223800081 2024-11-21T21:16:40,083 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T21:16:40,083 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732223800083"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732223800083"}]},"ts":"1732223800083"} 2024-11-21T21:16:40,085 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T21:16:40,087 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T21:16:40,087 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732223800087"}]},"ts":"1732223800087"} 2024-11-21T21:16:40,089 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T21:16:40,090 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5a7f333184706b95cbb8c49fb3d84f31, ASSIGN}] 2024-11-21T21:16:40,091 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5a7f333184706b95cbb8c49fb3d84f31, ASSIGN 2024-11-21T21:16:40,093 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5a7f333184706b95cbb8c49fb3d84f31, ASSIGN; state=OFFLINE, location=849dee24a242,41213,1732223653469; forceNewPlan=false, retain=false 2024-11-21T21:16:40,244 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=5a7f333184706b95cbb8c49fb3d84f31, regionState=OPENING, regionLocation=849dee24a242,41213,1732223653469 2024-11-21T21:16:40,248 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5a7f333184706b95cbb8c49fb3d84f31, ASSIGN because future has completed 2024-11-21T21:16:40,248 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469}] 2024-11-21T21:16:40,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-21T21:16:40,408 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:40,408 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(7752): Opening region: {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} 2024-11-21T21:16:40,409 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:40,409 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:16:40,409 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(7794): checking encryption for 5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:40,409 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(7797): checking classloading for 5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:40,411 INFO [StoreOpener-5a7f333184706b95cbb8c49fb3d84f31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:40,412 INFO [StoreOpener-5a7f333184706b95cbb8c49fb3d84f31-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T21:16:40,412 INFO [StoreOpener-5a7f333184706b95cbb8c49fb3d84f31-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5a7f333184706b95cbb8c49fb3d84f31 columnFamilyName A 2024-11-21T21:16:40,412 DEBUG [StoreOpener-5a7f333184706b95cbb8c49fb3d84f31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:16:40,413 INFO [StoreOpener-5a7f333184706b95cbb8c49fb3d84f31-1 {}] regionserver.HStore(327): Store=5a7f333184706b95cbb8c49fb3d84f31/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:16:40,413 INFO [StoreOpener-5a7f333184706b95cbb8c49fb3d84f31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:40,414 INFO [StoreOpener-5a7f333184706b95cbb8c49fb3d84f31-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T21:16:40,415 INFO [StoreOpener-5a7f333184706b95cbb8c49fb3d84f31-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5a7f333184706b95cbb8c49fb3d84f31 columnFamilyName B 2024-11-21T21:16:40,415 DEBUG [StoreOpener-5a7f333184706b95cbb8c49fb3d84f31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:16:40,415 INFO [StoreOpener-5a7f333184706b95cbb8c49fb3d84f31-1 {}] regionserver.HStore(327): Store=5a7f333184706b95cbb8c49fb3d84f31/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:16:40,415 INFO [StoreOpener-5a7f333184706b95cbb8c49fb3d84f31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:40,417 INFO [StoreOpener-5a7f333184706b95cbb8c49fb3d84f31-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T21:16:40,417 INFO [StoreOpener-5a7f333184706b95cbb8c49fb3d84f31-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5a7f333184706b95cbb8c49fb3d84f31 columnFamilyName C 2024-11-21T21:16:40,417 DEBUG [StoreOpener-5a7f333184706b95cbb8c49fb3d84f31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:16:40,417 INFO [StoreOpener-5a7f333184706b95cbb8c49fb3d84f31-1 {}] regionserver.HStore(327): Store=5a7f333184706b95cbb8c49fb3d84f31/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:16:40,418 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:40,418 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1038): replaying wal for 5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:40,419 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:40,419 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:40,421 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1048): stopping wal replay for 5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:40,421 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1060): Cleaning up temporary data for 5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:40,422 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T21:16:40,424 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1093): writing seq id for 5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:40,427 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T21:16:40,427 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1114): Opened 5a7f333184706b95cbb8c49fb3d84f31; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75003511, jitterRate=0.11763940751552582}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T21:16:40,427 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:40,429 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1006): Region open journal for 5a7f333184706b95cbb8c49fb3d84f31: Running coprocessor pre-open hook at 1732223800409Writing region info on filesystem at 1732223800409Initializing all the Stores at 1732223800410 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732223800410Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732223800410Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732223800410Cleaning up temporary data from old regions at 1732223800421 (+11 ms)Running coprocessor post-open hooks at 1732223800427 (+6 ms)Region opened successfully at 1732223800428 (+1 ms) 2024-11-21T21:16:40,430 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., pid=152, masterSystemTime=1732223800401 2024-11-21T21:16:40,432 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:40,432 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:40,433 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=5a7f333184706b95cbb8c49fb3d84f31, regionState=OPEN, openSeqNum=2, regionLocation=849dee24a242,41213,1732223653469 2024-11-21T21:16:40,434 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=152, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 because future has completed 2024-11-21T21:16:40,436 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-21T21:16:40,437 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; OpenRegionProcedure 5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 in 187 msec 2024-11-21T21:16:40,438 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-21T21:16:40,438 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5a7f333184706b95cbb8c49fb3d84f31, ASSIGN in 347 msec 2024-11-21T21:16:40,439 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T21:16:40,439 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732223800439"}]},"ts":"1732223800439"} 2024-11-21T21:16:40,440 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T21:16:40,441 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T21:16:40,442 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 783 msec 2024-11-21T21:16:40,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-21T21:16:40,802 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:16:40,805 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a769332, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:40,805 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:16:40,805 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:16:40,807 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:16:40,807 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:16:40,807 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:16:40,807 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@782dc32, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:40,807 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:16:40,807 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:16:40,808 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:40,808 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:16:40,809 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44d7702e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:40,809 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:16:40,810 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:16:40,810 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:16:40,811 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42920, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:16:40,812 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T21:16:40,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 849dee24a242,42485,1732223652547 2024-11-21T21:16:40,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@db21fdd 2024-11-21T21:16:40,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T21:16:40,814 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40806, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T21:16:40,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T21:16:40,816 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T21:16:40,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=153, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-21T21:16:40,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742377_1553 (size=1097) 2024-11-21T21:16:41,232 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-21T21:16:41,233 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-21T21:16:41,238 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T21:16:41,243 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5a7f333184706b95cbb8c49fb3d84f31, REOPEN/MOVE}] 2024-11-21T21:16:41,245 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5a7f333184706b95cbb8c49fb3d84f31, REOPEN/MOVE 2024-11-21T21:16:41,246 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=155 updating hbase:meta row=5a7f333184706b95cbb8c49fb3d84f31, regionState=CLOSING, regionLocation=849dee24a242,41213,1732223653469 2024-11-21T21:16:41,248 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5a7f333184706b95cbb8c49fb3d84f31, REOPEN/MOVE because future has completed 2024-11-21T21:16:41,248 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T21:16:41,248 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=156, ppid=155, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469}] 2024-11-21T21:16:41,401 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] handler.UnassignRegionHandler(122): Close 5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:41,402 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T21:16:41,402 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1722): Closing 5a7f333184706b95cbb8c49fb3d84f31, disabling compactions & flushes 2024-11-21T21:16:41,402 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:41,402 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:41,402 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. after waiting 0 ms 2024-11-21T21:16:41,402 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:41,407 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T21:16:41,408 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:41,408 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1676): Region close journal for 5a7f333184706b95cbb8c49fb3d84f31: Waiting for close lock at 1732223801402Running coprocessor pre-close hooks at 1732223801402Disabling compacts and flushes for region at 1732223801402Disabling writes for close at 1732223801402Writing region close event to WAL at 1732223801402Running coprocessor post-close hooks at 1732223801408 (+6 ms)Closed at 1732223801408 2024-11-21T21:16:41,408 WARN [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegionServer(3299): Not adding moved region record: 5a7f333184706b95cbb8c49fb3d84f31 to self. 2024-11-21T21:16:41,410 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] handler.UnassignRegionHandler(157): Closed 5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:41,411 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=155 updating hbase:meta row=5a7f333184706b95cbb8c49fb3d84f31, regionState=CLOSED 2024-11-21T21:16:41,412 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=156, ppid=155, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 because future has completed 2024-11-21T21:16:41,415 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=156, resume processing ppid=155 2024-11-21T21:16:41,415 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, ppid=155, state=SUCCESS, hasLock=false; CloseRegionProcedure 5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 in 165 msec 2024-11-21T21:16:41,415 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5a7f333184706b95cbb8c49fb3d84f31, REOPEN/MOVE; state=CLOSED, location=849dee24a242,41213,1732223653469; forceNewPlan=false, retain=true 2024-11-21T21:16:41,566 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=155 updating hbase:meta row=5a7f333184706b95cbb8c49fb3d84f31, regionState=OPENING, regionLocation=849dee24a242,41213,1732223653469 2024-11-21T21:16:41,570 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5a7f333184706b95cbb8c49fb3d84f31, REOPEN/MOVE because future has completed 2024-11-21T21:16:41,571 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=155, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469}] 2024-11-21T21:16:41,604 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T21:16:41,732 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:41,732 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(7752): Opening region: {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} 2024-11-21T21:16:41,732 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:41,732 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T21:16:41,732 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(7794): checking encryption for 5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:41,732 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(7797): checking classloading for 5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:41,734 INFO [StoreOpener-5a7f333184706b95cbb8c49fb3d84f31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:41,735 INFO [StoreOpener-5a7f333184706b95cbb8c49fb3d84f31-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T21:16:41,735 INFO [StoreOpener-5a7f333184706b95cbb8c49fb3d84f31-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5a7f333184706b95cbb8c49fb3d84f31 columnFamilyName A 2024-11-21T21:16:41,737 DEBUG [StoreOpener-5a7f333184706b95cbb8c49fb3d84f31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:16:41,737 INFO [StoreOpener-5a7f333184706b95cbb8c49fb3d84f31-1 {}] regionserver.HStore(327): Store=5a7f333184706b95cbb8c49fb3d84f31/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:16:41,738 INFO [StoreOpener-5a7f333184706b95cbb8c49fb3d84f31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:41,738 INFO [StoreOpener-5a7f333184706b95cbb8c49fb3d84f31-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T21:16:41,739 INFO [StoreOpener-5a7f333184706b95cbb8c49fb3d84f31-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5a7f333184706b95cbb8c49fb3d84f31 columnFamilyName B 2024-11-21T21:16:41,739 DEBUG [StoreOpener-5a7f333184706b95cbb8c49fb3d84f31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:16:41,739 INFO [StoreOpener-5a7f333184706b95cbb8c49fb3d84f31-1 {}] regionserver.HStore(327): Store=5a7f333184706b95cbb8c49fb3d84f31/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:16:41,739 INFO [StoreOpener-5a7f333184706b95cbb8c49fb3d84f31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:41,740 INFO [StoreOpener-5a7f333184706b95cbb8c49fb3d84f31-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-21T21:16:41,740 INFO [StoreOpener-5a7f333184706b95cbb8c49fb3d84f31-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5a7f333184706b95cbb8c49fb3d84f31 columnFamilyName C 2024-11-21T21:16:41,740 DEBUG [StoreOpener-5a7f333184706b95cbb8c49fb3d84f31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:16:41,741 INFO [StoreOpener-5a7f333184706b95cbb8c49fb3d84f31-1 {}] regionserver.HStore(327): Store=5a7f333184706b95cbb8c49fb3d84f31/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T21:16:41,741 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:41,741 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1038): replaying wal for 5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:41,742 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:41,743 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:41,745 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1048): stopping wal replay for 5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:41,745 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1060): Cleaning up temporary data for 5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:41,746 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T21:16:41,748 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1093): writing seq id for 5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:41,749 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1114): Opened 5a7f333184706b95cbb8c49fb3d84f31; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65234376, jitterRate=-0.027932047843933105}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T21:16:41,749 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:41,749 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1006): Region open journal for 5a7f333184706b95cbb8c49fb3d84f31: Running coprocessor pre-open hook at 1732223801733Writing region info on filesystem at 1732223801733Initializing all the Stores at 1732223801733Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732223801733Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732223801734 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732223801734Cleaning up temporary data from old regions at 1732223801745 (+11 ms)Running coprocessor post-open hooks at 1732223801749 (+4 ms)Region opened successfully at 1732223801749 2024-11-21T21:16:41,750 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., pid=157, masterSystemTime=1732223801725 2024-11-21T21:16:41,752 DEBUG [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:41,752 INFO [RS_OPEN_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:41,752 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=155 updating hbase:meta row=5a7f333184706b95cbb8c49fb3d84f31, regionState=OPEN, openSeqNum=5, regionLocation=849dee24a242,41213,1732223653469 2024-11-21T21:16:41,753 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=157, ppid=155, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 because future has completed 2024-11-21T21:16:41,756 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=155 2024-11-21T21:16:41,756 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=155, state=SUCCESS, hasLock=false; OpenRegionProcedure 5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 in 183 msec 2024-11-21T21:16:41,758 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-21T21:16:41,758 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5a7f333184706b95cbb8c49fb3d84f31, REOPEN/MOVE in 513 msec 2024-11-21T21:16:41,762 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=154, resume processing ppid=153 2024-11-21T21:16:41,762 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, ppid=153, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 521 msec 2024-11-21T21:16:41,765 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 947 msec 2024-11-21T21:16:41,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153 2024-11-21T21:16:41,766 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:16:41,767 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bd32fa2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:41,767 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:16:41,767 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:16:41,768 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:16:41,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:16:41,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:16:41,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d28178e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:41,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:16:41,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:16:41,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:41,769 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40824, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:16:41,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fdd9e36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:41,771 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d29c9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:41,771 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:16:41,771 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:16:41,772 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:16:41,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:16:41,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:16:41,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12cf2b42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:41,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:16:41,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:16:41,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:41,773 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40842, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:16:41,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d6f9d9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:41,774 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1572f078, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:41,774 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:16:41,774 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:16:41,775 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:16:41,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:16:41,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:16:41,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bfe0d5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:41,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:16:41,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:16:41,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:41,777 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40854, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:16:41,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5328b93f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:41,778 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23448b6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:41,778 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:16:41,778 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:16:41,779 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:16:41,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:16:41,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:16:41,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2834dc37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:41,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:16:41,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:16:41,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:41,780 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40878, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:16:41,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@929a9c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:41,781 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d7db06e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:41,781 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:16:41,782 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:16:41,782 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:16:41,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:16:41,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:16:41,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b4e8434, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:41,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:16:41,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:16:41,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:41,784 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40892, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:16:41,785 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bab7b70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:41,785 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41947491, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:41,785 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:16:41,786 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:16:41,786 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:16:41,786 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:16:41,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:16:41,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63d1db29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:41,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:16:41,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:16:41,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:41,788 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:16:41,789 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c6b9b9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:41,789 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5451f0cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:41,789 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:16:41,790 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:16:41,790 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:16:41,791 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:16:41,791 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:16:41,791 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b1af13d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:41,791 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:16:41,791 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:16:41,791 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:41,792 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40938, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:16:41,793 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76f0964e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:41,794 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b0e0fb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:41,794 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:16:41,794 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:16:41,795 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:16:41,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:16:41,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:16:41,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d743726, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:41,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:16:41,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:16:41,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:41,797 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40958, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:16:41,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f9479d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:41,798 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cbb8a41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:41,798 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:16:41,798 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:16:41,798 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:16:41,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:16:41,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:16:41,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@179dedd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:41,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:16:41,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:16:41,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:41,799 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40966, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:16:41,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2907308e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:41,800 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31a58f58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:41,800 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 849dee24a242,42485,-1 for getting cluster id 2024-11-21T21:16:41,800 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T21:16:41,801 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '237381a1-2648-400b-a825-c26d79780e33' 2024-11-21T21:16:41,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T21:16:41,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "237381a1-2648-400b-a825-c26d79780e33" 2024-11-21T21:16:41,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bd6fd42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:41,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [849dee24a242,42485,-1] 2024-11-21T21:16:41,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T21:16:41,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:16:41,802 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40980, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T21:16:41,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f284e21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T21:16:41,803 DEBUG [Thread-2471 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:16:41,803 DEBUG [Thread-2473 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:16:41,803 DEBUG [Thread-2464 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:16:41,803 DEBUG [Thread-2456 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:16:41,803 DEBUG [Thread-2462 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:16:41,803 DEBUG [Thread-2467 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:16:41,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:16:41,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:16:41,804 DEBUG [Thread-2460 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:16:41,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:16:41,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:16:41,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:16:41,804 DEBUG [Thread-2469 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:16:41,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:16:41,804 DEBUG [Thread-2462 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:16:41,804 DEBUG [Thread-2458 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:16:41,804 DEBUG [Thread-2462 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:16:41,804 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:16:41,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:16:41,805 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:16:41,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:16:41,805 DEBUG [Thread-2460 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:16:41,809 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42936, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:16:41,809 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42946, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:16:41,805 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42942, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:16:41,809 DEBUG [Thread-2460 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:16:41,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:16:41,810 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:16:41,810 DEBUG [Thread-2475 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T21:16:41,810 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:16:41,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:16:41,810 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:16:41,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=849dee24a242,41213,1732223653469, seqNum=-1] 2024-11-21T21:16:41,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T21:16:41,811 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42962, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:16:41,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5] 2024-11-21T21:16:41,812 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42986, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:16:41,812 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42950, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:16:41,812 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42996, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:16:41,812 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:16:41,812 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42984, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:16:41,813 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42998, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T21:16:41,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5] 2024-11-21T21:16:41,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5] 2024-11-21T21:16:41,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5] 2024-11-21T21:16:41,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5] 2024-11-21T21:16:41,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-21T21:16:41,814 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5] 2024-11-21T21:16:41,815 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5] 2024-11-21T21:16:41,815 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5] 2024-11-21T21:16:41,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-21T21:16:41,816 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:16:41,816 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5] 2024-11-21T21:16:41,816 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5] 2024-11-21T21:16:41,817 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:16:41,817 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:16:41,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:41,819 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5a7f333184706b95cbb8c49fb3d84f31 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-21T21:16:41,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=A 2024-11-21T21:16:41,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:41,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=B 2024-11-21T21:16:41,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:41,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=C 2024-11-21T21:16:41,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:41,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411212236ae25ee394813a23dd6013e712929_5a7f333184706b95cbb8c49fb3d84f31 is 50, key is test_row_0/A:col10/1732223801818/Put/seqid=0 2024-11-21T21:16:41,841 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9554, totalHeapSize=21976, min timestamp=1732223801831, max timestamp=1732223801839Num uniques 859; 2024-11-21T21:16:41,841 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39324, new segment data size=9554, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T21:16:41,842 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732223801831, max timestamp=1732223801839Num uniques 859; 2024-11-21T21:16:41,842 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732223801831, max timestamp=1732223801839Num uniques 859; 2024-11-21T21:16:41,842 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T21:16:41,842 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/849dee24a242:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-21T21:16:41,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742378_1554 (size=14594) 2024-11-21T21:16:41,850 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:16:41,853 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411212236ae25ee394813a23dd6013e712929_5a7f333184706b95cbb8c49fb3d84f31 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212236ae25ee394813a23dd6013e712929_5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:41,854 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/4343f0b9f95e46c2a1c3b2b14a698fa3, store: [table=TestAcidGuarantees family=A region=5a7f333184706b95cbb8c49fb3d84f31] 2024-11-21T21:16:41,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/4343f0b9f95e46c2a1c3b2b14a698fa3 is 175, key is test_row_0/A:col10/1732223801818/Put/seqid=0 2024-11-21T21:16:41,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:41,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223861858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:41,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:41,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223861858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:41,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:41,863 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:41,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42950 deadline: 1732223861859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:41,864 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:41,864 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:41,864 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:41,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:41,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223861860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:41,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:41,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:41,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:41,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:41,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223861863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:41,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:41,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:41,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:41,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742379_1555 (size=39549) 2024-11-21T21:16:41,874 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/4343f0b9f95e46c2a1c3b2b14a698fa3 2024-11-21T21:16:41,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/e4e2205110ec4855a7f9177b4a5a4769 is 50, key is test_row_0/B:col10/1732223801818/Put/seqid=0 2024-11-21T21:16:41,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742380_1556 (size=12001) 2024-11-21T21:16:41,902 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/e4e2205110ec4855a7f9177b4a5a4769 2024-11-21T21:16:41,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-21T21:16:41,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/03616e13e463456783f8dd38f7bb46a4 is 50, key is test_row_0/C:col10/1732223801818/Put/seqid=0 2024-11-21T21:16:41,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742381_1557 (size=12001) 2024-11-21T21:16:41,936 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/03616e13e463456783f8dd38f7bb46a4 2024-11-21T21:16:41,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/4343f0b9f95e46c2a1c3b2b14a698fa3 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/4343f0b9f95e46c2a1c3b2b14a698fa3 2024-11-21T21:16:41,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/4343f0b9f95e46c2a1c3b2b14a698fa3, entries=200, sequenceid=14, filesize=38.6 K 2024-11-21T21:16:41,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/e4e2205110ec4855a7f9177b4a5a4769 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/e4e2205110ec4855a7f9177b4a5a4769 2024-11-21T21:16:41,950 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/e4e2205110ec4855a7f9177b4a5a4769, entries=150, sequenceid=14, filesize=11.7 K 2024-11-21T21:16:41,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/03616e13e463456783f8dd38f7bb46a4 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/03616e13e463456783f8dd38f7bb46a4 2024-11-21T21:16:41,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/03616e13e463456783f8dd38f7bb46a4, entries=150, sequenceid=14, filesize=11.7 K 2024-11-21T21:16:41,954 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 5a7f333184706b95cbb8c49fb3d84f31 in 135ms, sequenceid=14, compaction requested=false 2024-11-21T21:16:41,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5a7f333184706b95cbb8c49fb3d84f31: 2024-11-21T21:16:41,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-21T21:16:41,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:41,971 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 5a7f333184706b95cbb8c49fb3d84f31 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-21T21:16:41,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=A 2024-11-21T21:16:41,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:16:41,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=B 2024-11-21T21:16:41,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:16:41,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=C 2024-11-21T21:16:41,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-21T21:16:41,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:41,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. as already flushing 2024-11-21T21:16:41,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411211ff8d56e501b43a4bdda6dd04a32a79f_5a7f333184706b95cbb8c49fb3d84f31 is 50, key is test_row_0/A:col10/1732223801858/Put/seqid=0 2024-11-21T21:16:41,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:41,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223861975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:41,981 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:41,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:41,981 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:41,981 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:41,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42950 deadline: 1732223861976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:41,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:41,981 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:41,981 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:41,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223861976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:41,981 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:41,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:41,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:41,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:41,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:41,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223861978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:41,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742382_1558 (size=12154) 2024-11-21T21:16:41,982 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:41,982 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:41,982 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:41,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:16:41,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:41,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223861981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:41,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:41,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:41,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:41,986 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411211ff8d56e501b43a4bdda6dd04a32a79f_5a7f333184706b95cbb8c49fb3d84f31 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411211ff8d56e501b43a4bdda6dd04a32a79f_5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:41,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/bf6bbbb7668248e3a5a9ca45f58891ad, store: [table=TestAcidGuarantees family=A region=5a7f333184706b95cbb8c49fb3d84f31] 2024-11-21T21:16:41,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/bf6bbbb7668248e3a5a9ca45f58891ad is 175, key is test_row_0/A:col10/1732223801858/Put/seqid=0 2024-11-21T21:16:41,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742383_1559 (size=30955) 2024-11-21T21:16:41,995 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/bf6bbbb7668248e3a5a9ca45f58891ad 2024-11-21T21:16:42,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/79bd7201ea7c4469a6154524bdff8ca8 is 50, key is test_row_0/B:col10/1732223801858/Put/seqid=0 2024-11-21T21:16:42,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742384_1560 (size=12001) 2024-11-21T21:16:42,018 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/79bd7201ea7c4469a6154524bdff8ca8 2024-11-21T21:16:42,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/02bb581151d64d4fa2559aaf82c295c0 is 50, key is test_row_0/C:col10/1732223801858/Put/seqid=0 2024-11-21T21:16:42,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742385_1561 (size=12001) 2024-11-21T21:16:42,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:42,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223862090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:42,093 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:42,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:42,093 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:42,093 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:42,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42950 deadline: 1732223862090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:42,094 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:42,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:42,094 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:42,094 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:42,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223862090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:42,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:42,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:42,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:42,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:42,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223862090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:42,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:42,094 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:42,094 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:42,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223862091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:42,094 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:42,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:42,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:42,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:42,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-21T21:16:42,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:42,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223862300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:42,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:42,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:42,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223862301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:42,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:42,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:42,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:42,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:42,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:42,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42950 deadline: 1732223862301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:42,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:42,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:42,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223862301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:42,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:42,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:42,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:42,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:42,304 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:42,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223862301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:42,304 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:42,304 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:42,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:42,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:42,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:42,428 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/02bb581151d64d4fa2559aaf82c295c0 2024-11-21T21:16:42,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/bf6bbbb7668248e3a5a9ca45f58891ad as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/bf6bbbb7668248e3a5a9ca45f58891ad 2024-11-21T21:16:42,434 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/bf6bbbb7668248e3a5a9ca45f58891ad, entries=150, sequenceid=50, filesize=30.2 K 2024-11-21T21:16:42,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/79bd7201ea7c4469a6154524bdff8ca8 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/79bd7201ea7c4469a6154524bdff8ca8 2024-11-21T21:16:42,438 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/79bd7201ea7c4469a6154524bdff8ca8, entries=150, sequenceid=50, filesize=11.7 K 2024-11-21T21:16:42,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/02bb581151d64d4fa2559aaf82c295c0 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/02bb581151d64d4fa2559aaf82c295c0 2024-11-21T21:16:42,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-21T21:16:42,441 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/02bb581151d64d4fa2559aaf82c295c0, entries=150, sequenceid=50, filesize=11.7 K 2024-11-21T21:16:42,441 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 5a7f333184706b95cbb8c49fb3d84f31 in 470ms, sequenceid=50, compaction requested=false 2024-11-21T21:16:42,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 5a7f333184706b95cbb8c49fb3d84f31: 2024-11-21T21:16:42,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:42,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-21T21:16:42,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-21T21:16:42,444 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-21T21:16:42,444 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 625 msec 2024-11-21T21:16:42,445 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 632 msec 2024-11-21T21:16:42,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:42,613 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5a7f333184706b95cbb8c49fb3d84f31 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-21T21:16:42,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=A 2024-11-21T21:16:42,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:42,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=B 2024-11-21T21:16:42,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:42,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=C 2024-11-21T21:16:42,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:42,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ba5efa6366a4472f8f1f1180aa848c70_5a7f333184706b95cbb8c49fb3d84f31 is 50, key is test_row_0/A:col10/1732223802613/Put/seqid=0 2024-11-21T21:16:42,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742386_1562 (size=12154) 2024-11-21T21:16:42,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:42,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223862627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:42,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:42,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:42,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:42,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:42,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223862632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:42,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:42,639 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:42,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42950 deadline: 1732223862633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:42,639 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:42,639 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:42,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:42,639 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:42,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223862634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:42,639 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:42,639 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:42,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:42,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223862634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:42,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:42,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:42,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:42,639 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:42,639 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:42,639 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:42,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:42,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223862740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:42,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:42,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:42,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:42,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:42,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42950 deadline: 1732223862740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:42,742 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:42,742 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:42,743 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:42,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:42,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223862750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:42,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:42,752 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:42,752 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:42,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223862750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:42,752 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:42,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:42,753 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:42,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223862750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:42,753 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:42,753 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:42,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:42,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:42,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:42,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-21T21:16:42,950 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:16:42,950 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:16:42,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-21T21:16:42,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-21T21:16:42,952 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:16:42,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:42,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223862950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:42,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:42,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42950 deadline: 1732223862950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:42,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:42,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:42,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:42,953 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:42,953 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:42,953 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:42,953 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:16:42,953 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:16:42,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:42,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223862960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:42,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:42,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:42,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:42,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223862960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:42,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:42,962 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:42,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:42,962 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:42,962 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:42,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223862960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:42,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:42,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:42,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:43,023 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:16:43,026 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ba5efa6366a4472f8f1f1180aa848c70_5a7f333184706b95cbb8c49fb3d84f31 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ba5efa6366a4472f8f1f1180aa848c70_5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:43,026 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/f785e28c91604991a62412b2f87f110f, store: [table=TestAcidGuarantees family=A region=5a7f333184706b95cbb8c49fb3d84f31] 2024-11-21T21:16:43,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/f785e28c91604991a62412b2f87f110f is 175, key is test_row_0/A:col10/1732223802613/Put/seqid=0 2024-11-21T21:16:43,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742387_1563 (size=30955) 2024-11-21T21:16:43,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-21T21:16:43,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-21T21:16:43,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:43,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. as already flushing 2024-11-21T21:16:43,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:43,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:43,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:43,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:43,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-21T21:16:43,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:43,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. as already flushing 2024-11-21T21:16:43,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:43,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:43,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:43,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:43,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:43,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42950 deadline: 1732223863260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:43,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:43,261 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:43,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223863260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:43,261 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:43,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:43,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:43,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:43,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:43,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-21T21:16:43,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:43,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223863270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:43,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:43,273 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:43,273 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:43,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223863270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:43,273 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:43,273 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:43,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:43,273 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:43,273 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:43,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223863270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:43,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:43,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:43,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:43,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-21T21:16:43,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:43,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. as already flushing 2024-11-21T21:16:43,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:43,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:43,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:43,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:43,442 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/f785e28c91604991a62412b2f87f110f 2024-11-21T21:16:43,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/1c0e70eb910e4402b0d897461d8f539e is 50, key is test_row_0/B:col10/1732223802613/Put/seqid=0 2024-11-21T21:16:43,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742388_1564 (size=12001) 2024-11-21T21:16:43,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-21T21:16:43,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:43,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. as already flushing 2024-11-21T21:16:43,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:43,563 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:43,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:43,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:43,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-21T21:16:43,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-21T21:16:43,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:43,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. as already flushing 2024-11-21T21:16:43,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:43,716 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:43,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:43,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:43,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:43,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42950 deadline: 1732223863770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:43,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:43,773 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:43,773 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:43,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223863770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:43,773 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:43,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:43,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:43,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:43,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:43,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223863780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:43,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:43,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223863780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:43,783 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:43,783 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:43,783 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:43,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:43,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:43,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:43,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:43,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223863780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:43,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:43,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:43,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:43,851 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/1c0e70eb910e4402b0d897461d8f539e 2024-11-21T21:16:43,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/19f425bf2ed74bf89aa4ddba9acfa1f8 is 50, key is test_row_0/C:col10/1732223802613/Put/seqid=0 2024-11-21T21:16:43,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742389_1565 (size=12001) 2024-11-21T21:16:43,862 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/19f425bf2ed74bf89aa4ddba9acfa1f8 2024-11-21T21:16:43,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/f785e28c91604991a62412b2f87f110f as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/f785e28c91604991a62412b2f87f110f 2024-11-21T21:16:43,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/f785e28c91604991a62412b2f87f110f, entries=150, sequenceid=62, filesize=30.2 K 2024-11-21T21:16:43,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-21T21:16:43,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:43,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. as already flushing 2024-11-21T21:16:43,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:43,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/1c0e70eb910e4402b0d897461d8f539e as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/1c0e70eb910e4402b0d897461d8f539e 2024-11-21T21:16:43,868 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:43,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:43,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:43,872 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/1c0e70eb910e4402b0d897461d8f539e, entries=150, sequenceid=62, filesize=11.7 K 2024-11-21T21:16:43,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/19f425bf2ed74bf89aa4ddba9acfa1f8 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/19f425bf2ed74bf89aa4ddba9acfa1f8 2024-11-21T21:16:43,875 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/19f425bf2ed74bf89aa4ddba9acfa1f8, entries=150, sequenceid=62, filesize=11.7 K 2024-11-21T21:16:43,876 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 5a7f333184706b95cbb8c49fb3d84f31 in 1263ms, sequenceid=62, compaction requested=true 2024-11-21T21:16:43,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5a7f333184706b95cbb8c49fb3d84f31: 2024-11-21T21:16:43,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a7f333184706b95cbb8c49fb3d84f31:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:16:43,876 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:16:43,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:43,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a7f333184706b95cbb8c49fb3d84f31:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:16:43,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:43,876 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:16:43,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a7f333184706b95cbb8c49fb3d84f31:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:16:43,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:16:43,877 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:16:43,877 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:16:43,877 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): 5a7f333184706b95cbb8c49fb3d84f31/A is initiating minor compaction (all files) 2024-11-21T21:16:43,877 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): 5a7f333184706b95cbb8c49fb3d84f31/B is initiating minor compaction (all files) 2024-11-21T21:16:43,877 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a7f333184706b95cbb8c49fb3d84f31/A in TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:43,877 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a7f333184706b95cbb8c49fb3d84f31/B in TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:43,877 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/e4e2205110ec4855a7f9177b4a5a4769, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/79bd7201ea7c4469a6154524bdff8ca8, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/1c0e70eb910e4402b0d897461d8f539e] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp, totalSize=35.2 K 2024-11-21T21:16:43,877 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/4343f0b9f95e46c2a1c3b2b14a698fa3, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/bf6bbbb7668248e3a5a9ca45f58891ad, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/f785e28c91604991a62412b2f87f110f] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp, totalSize=99.1 K 2024-11-21T21:16:43,877 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:43,877 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. files: [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/4343f0b9f95e46c2a1c3b2b14a698fa3, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/bf6bbbb7668248e3a5a9ca45f58891ad, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/f785e28c91604991a62412b2f87f110f] 2024-11-21T21:16:43,877 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting e4e2205110ec4855a7f9177b4a5a4769, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732223801815 2024-11-21T21:16:43,877 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4343f0b9f95e46c2a1c3b2b14a698fa3, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732223801815 2024-11-21T21:16:43,878 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 79bd7201ea7c4469a6154524bdff8ca8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732223801858 2024-11-21T21:16:43,878 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf6bbbb7668248e3a5a9ca45f58891ad, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732223801858 2024-11-21T21:16:43,878 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c0e70eb910e4402b0d897461d8f539e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732223801974 2024-11-21T21:16:43,878 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting f785e28c91604991a62412b2f87f110f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732223801974 2024-11-21T21:16:43,883 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5a7f333184706b95cbb8c49fb3d84f31] 2024-11-21T21:16:43,884 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a7f333184706b95cbb8c49fb3d84f31#B#compaction#491 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:16:43,884 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112173b571f8146a4da68beb681a766a52c2_5a7f333184706b95cbb8c49fb3d84f31 store=[table=TestAcidGuarantees family=A region=5a7f333184706b95cbb8c49fb3d84f31] 2024-11-21T21:16:43,884 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/9763ad8ab1e34eb9b6f76129cb3c26f1 is 50, key is test_row_0/B:col10/1732223802613/Put/seqid=0 2024-11-21T21:16:43,886 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112173b571f8146a4da68beb681a766a52c2_5a7f333184706b95cbb8c49fb3d84f31, store=[table=TestAcidGuarantees family=A region=5a7f333184706b95cbb8c49fb3d84f31] 2024-11-21T21:16:43,887 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112173b571f8146a4da68beb681a766a52c2_5a7f333184706b95cbb8c49fb3d84f31 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5a7f333184706b95cbb8c49fb3d84f31] 2024-11-21T21:16:43,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742390_1566 (size=12104) 2024-11-21T21:16:43,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742391_1567 (size=4469) 2024-11-21T21:16:44,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-21T21:16:44,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:44,021 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 5a7f333184706b95cbb8c49fb3d84f31 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-21T21:16:44,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=A 2024-11-21T21:16:44,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:44,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=B 2024-11-21T21:16:44,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:44,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=C 2024-11-21T21:16:44,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:44,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121f0ef9a92aaa5427ab7cd5bd2c42717dc_5a7f333184706b95cbb8c49fb3d84f31 is 50, key is test_row_0/A:col10/1732223802633/Put/seqid=0 2024-11-21T21:16:44,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742392_1568 (size=12154) 2024-11-21T21:16:44,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-21T21:16:44,291 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a7f333184706b95cbb8c49fb3d84f31#A#compaction#492 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:16:44,291 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/2f5e69d5955e4903a30cf68ea0aafe9d is 175, key is test_row_0/A:col10/1732223802613/Put/seqid=0 2024-11-21T21:16:44,294 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/9763ad8ab1e34eb9b6f76129cb3c26f1 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/9763ad8ab1e34eb9b6f76129cb3c26f1 2024-11-21T21:16:44,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742393_1569 (size=31058) 2024-11-21T21:16:44,298 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a7f333184706b95cbb8c49fb3d84f31/B of 5a7f333184706b95cbb8c49fb3d84f31 into 9763ad8ab1e34eb9b6f76129cb3c26f1(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:16:44,298 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a7f333184706b95cbb8c49fb3d84f31: 2024-11-21T21:16:44,298 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., storeName=5a7f333184706b95cbb8c49fb3d84f31/B, priority=13, startTime=1732223803876; duration=0sec 2024-11-21T21:16:44,298 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:16:44,298 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a7f333184706b95cbb8c49fb3d84f31:B 2024-11-21T21:16:44,298 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:16:44,299 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:16:44,299 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): 5a7f333184706b95cbb8c49fb3d84f31/C is initiating minor compaction (all files) 2024-11-21T21:16:44,299 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a7f333184706b95cbb8c49fb3d84f31/C in TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:44,299 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/03616e13e463456783f8dd38f7bb46a4, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/02bb581151d64d4fa2559aaf82c295c0, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/19f425bf2ed74bf89aa4ddba9acfa1f8] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp, totalSize=35.2 K 2024-11-21T21:16:44,299 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 03616e13e463456783f8dd38f7bb46a4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732223801815 2024-11-21T21:16:44,300 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 02bb581151d64d4fa2559aaf82c295c0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732223801858 2024-11-21T21:16:44,300 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 19f425bf2ed74bf89aa4ddba9acfa1f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732223801974 2024-11-21T21:16:44,304 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a7f333184706b95cbb8c49fb3d84f31#C#compaction#494 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:16:44,304 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/c4c0042840664f368aa2766ef2c6cd33 is 50, key is test_row_0/C:col10/1732223802613/Put/seqid=0 2024-11-21T21:16:44,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742394_1570 (size=12104) 2024-11-21T21:16:44,311 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/c4c0042840664f368aa2766ef2c6cd33 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/c4c0042840664f368aa2766ef2c6cd33 2024-11-21T21:16:44,315 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a7f333184706b95cbb8c49fb3d84f31/C of 5a7f333184706b95cbb8c49fb3d84f31 into c4c0042840664f368aa2766ef2c6cd33(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:16:44,315 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a7f333184706b95cbb8c49fb3d84f31: 2024-11-21T21:16:44,315 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., storeName=5a7f333184706b95cbb8c49fb3d84f31/C, priority=13, startTime=1732223803876; duration=0sec 2024-11-21T21:16:44,315 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:44,315 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a7f333184706b95cbb8c49fb3d84f31:C 2024-11-21T21:16:44,366 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T21:16:44,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:16:44,433 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121f0ef9a92aaa5427ab7cd5bd2c42717dc_5a7f333184706b95cbb8c49fb3d84f31 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121f0ef9a92aaa5427ab7cd5bd2c42717dc_5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:44,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/948454736c674488a24016b9fc20517a, store: [table=TestAcidGuarantees family=A region=5a7f333184706b95cbb8c49fb3d84f31] 2024-11-21T21:16:44,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/948454736c674488a24016b9fc20517a is 175, key is test_row_0/A:col10/1732223802633/Put/seqid=0 2024-11-21T21:16:44,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742395_1571 (size=30955) 2024-11-21T21:16:44,700 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/2f5e69d5955e4903a30cf68ea0aafe9d as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/2f5e69d5955e4903a30cf68ea0aafe9d 2024-11-21T21:16:44,704 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a7f333184706b95cbb8c49fb3d84f31/A of 5a7f333184706b95cbb8c49fb3d84f31 into 2f5e69d5955e4903a30cf68ea0aafe9d(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:16:44,704 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a7f333184706b95cbb8c49fb3d84f31: 2024-11-21T21:16:44,704 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., storeName=5a7f333184706b95cbb8c49fb3d84f31/A, priority=13, startTime=1732223803876; duration=0sec 2024-11-21T21:16:44,704 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:44,704 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a7f333184706b95cbb8c49fb3d84f31:A 2024-11-21T21:16:44,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:44,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. as already flushing 2024-11-21T21:16:44,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:44,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42950 deadline: 1732223864793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:44,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:44,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:44,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:44,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:44,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223864795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:44,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:44,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:44,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:44,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:44,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223864796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:44,801 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:44,801 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:44,801 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:44,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:44,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223864797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:44,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:44,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:44,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:44,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:44,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223864797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:44,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:44,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:44,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:44,846 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=82, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/948454736c674488a24016b9fc20517a 2024-11-21T21:16:44,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/a6a4399ad3a845449361005bacedccb2 is 50, key is test_row_0/B:col10/1732223802633/Put/seqid=0 2024-11-21T21:16:44,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742396_1572 (size=12001) 2024-11-21T21:16:44,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:44,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42950 deadline: 1732223864900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:44,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:44,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:44,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:44,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:44,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223864910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:44,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:44,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223864910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:44,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:44,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:44,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:44,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:44,913 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:44,913 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:44,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223864910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:44,913 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:44,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:44,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:44,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223864910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:44,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:44,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:44,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:44,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:44,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:45,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-21T21:16:45,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:45,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42950 deadline: 1732223865110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:45,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:45,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:45,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:45,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:45,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223865120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:45,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:45,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:45,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:45,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:45,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223865121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:45,123 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:45,123 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:45,123 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:45,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:45,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223865127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:45,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:45,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:45,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:45,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:45,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223865127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:45,130 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:45,130 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:45,130 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:45,255 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/a6a4399ad3a845449361005bacedccb2 2024-11-21T21:16:45,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/17b7277b32c345b0a9563814c5a0e621 is 50, key is test_row_0/C:col10/1732223802633/Put/seqid=0 2024-11-21T21:16:45,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742397_1573 (size=12001) 2024-11-21T21:16:45,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:45,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42950 deadline: 1732223865420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:45,421 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:45,421 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:45,421 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:45,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:45,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223865430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:45,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:45,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:45,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:45,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:45,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223865430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:45,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:45,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:45,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:45,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:45,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223865440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:45,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:45,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:45,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:45,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:45,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223865440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:45,443 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:45,443 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:45,443 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:45,664 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/17b7277b32c345b0a9563814c5a0e621 2024-11-21T21:16:45,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/948454736c674488a24016b9fc20517a as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/948454736c674488a24016b9fc20517a 2024-11-21T21:16:45,670 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/948454736c674488a24016b9fc20517a, entries=150, sequenceid=82, filesize=30.2 K 2024-11-21T21:16:45,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/a6a4399ad3a845449361005bacedccb2 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/a6a4399ad3a845449361005bacedccb2 2024-11-21T21:16:45,673 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/a6a4399ad3a845449361005bacedccb2, entries=150, sequenceid=82, filesize=11.7 K 2024-11-21T21:16:45,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/17b7277b32c345b0a9563814c5a0e621 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/17b7277b32c345b0a9563814c5a0e621 2024-11-21T21:16:45,676 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/17b7277b32c345b0a9563814c5a0e621, entries=150, sequenceid=82, filesize=11.7 K 2024-11-21T21:16:45,676 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=73.80 KB/75570 for 5a7f333184706b95cbb8c49fb3d84f31 in 1655ms, sequenceid=82, compaction requested=false 2024-11-21T21:16:45,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 5a7f333184706b95cbb8c49fb3d84f31: 2024-11-21T21:16:45,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:45,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-21T21:16:45,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-21T21:16:45,679 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-21T21:16:45,679 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7250 sec 2024-11-21T21:16:45,680 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.7290 sec 2024-11-21T21:16:45,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:45,931 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5a7f333184706b95cbb8c49fb3d84f31 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-21T21:16:45,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=A 2024-11-21T21:16:45,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:45,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=B 2024-11-21T21:16:45,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:45,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=C 2024-11-21T21:16:45,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:45,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a6bb73b7cb3a4af4aea030c25520a20b_5a7f333184706b95cbb8c49fb3d84f31 is 50, key is test_row_0/A:col10/1732223804795/Put/seqid=0 2024-11-21T21:16:45,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742398_1574 (size=14594) 2024-11-21T21:16:45,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:45,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223865955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:45,960 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:45,960 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:45,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:45,960 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:45,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42950 deadline: 1732223865955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:45,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:45,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:45,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:45,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:45,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223865956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:45,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:45,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:45,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:45,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:45,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223865956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:45,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:45,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:45,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:45,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:45,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223865959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:45,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:45,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:45,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:46,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:46,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223866070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:46,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:46,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42950 deadline: 1732223866070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:46,073 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:46,073 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:46,073 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:46,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:46,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:46,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223866070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:46,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:46,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:46,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:46,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:46,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:46,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:46,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223866071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:46,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:46,073 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:46,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223866071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:46,073 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:46,073 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:46,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:46,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:46,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:46,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:46,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223866280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:46,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:46,283 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:46,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42950 deadline: 1732223866280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:46,283 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:46,284 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:46,284 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:46,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:46,284 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:46,284 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:46,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223866280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:46,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:46,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:46,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:46,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:46,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223866281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:46,285 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:46,285 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:46,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:46,285 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:46,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223866281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:46,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:46,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:46,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:46,345 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:16:46,347 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a6bb73b7cb3a4af4aea030c25520a20b_5a7f333184706b95cbb8c49fb3d84f31 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a6bb73b7cb3a4af4aea030c25520a20b_5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:46,348 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/5d9d2ed2bf3142199f6b50af11c3bf32, store: [table=TestAcidGuarantees family=A region=5a7f333184706b95cbb8c49fb3d84f31] 2024-11-21T21:16:46,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/5d9d2ed2bf3142199f6b50af11c3bf32 is 175, key is test_row_0/A:col10/1732223804795/Put/seqid=0 2024-11-21T21:16:46,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742399_1575 (size=39549) 2024-11-21T21:16:46,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:46,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223866590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:46,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:46,592 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:46,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42950 deadline: 1732223866590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:46,592 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:46,592 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:46,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:46,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:46,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:46,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:46,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223866590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:46,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:46,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:46,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:46,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:46,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223866590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:46,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:46,593 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:46,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223866590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:46,593 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:46,593 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:46,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:46,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:46,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:46,752 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=100, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/5d9d2ed2bf3142199f6b50af11c3bf32 2024-11-21T21:16:46,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/ec8dde1ab3a943a6b6d6821abbc0ed54 is 50, key is test_row_0/B:col10/1732223804795/Put/seqid=0 2024-11-21T21:16:46,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742400_1576 (size=12001) 2024-11-21T21:16:47,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:47,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223867100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:47,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:47,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:47,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42950 deadline: 1732223867100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:47,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:47,103 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:47,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:47,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:47,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223867100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:47,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:47,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:47,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:47,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:47,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:47,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223867100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:47,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:47,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:47,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:47,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223867100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:47,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:47,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:47,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:47,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:47,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:47,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-21T21:16:47,111 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:16:47,111 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:16:47,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-21T21:16:47,112 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:16:47,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-21T21:16:47,113 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:16:47,113 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:16:47,160 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/ec8dde1ab3a943a6b6d6821abbc0ed54 2024-11-21T21:16:47,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/4758f5d824744ae2aeddbc496eccafbc is 50, key is test_row_0/C:col10/1732223804795/Put/seqid=0 2024-11-21T21:16:47,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742401_1577 (size=12001) 2024-11-21T21:16:47,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-21T21:16:47,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-21T21:16:47,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:47,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. as already flushing 2024-11-21T21:16:47,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:47,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:47,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:47,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:47,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-21T21:16:47,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:47,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. as already flushing 2024-11-21T21:16:47,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:47,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:47,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:47,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:47,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-21T21:16:47,568 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/4758f5d824744ae2aeddbc496eccafbc 2024-11-21T21:16:47,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-21T21:16:47,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:47,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. as already flushing 2024-11-21T21:16:47,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:47,570 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:47,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:47,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:47,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/5d9d2ed2bf3142199f6b50af11c3bf32 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/5d9d2ed2bf3142199f6b50af11c3bf32 2024-11-21T21:16:47,577 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/5d9d2ed2bf3142199f6b50af11c3bf32, entries=200, sequenceid=100, filesize=38.6 K 2024-11-21T21:16:47,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/ec8dde1ab3a943a6b6d6821abbc0ed54 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/ec8dde1ab3a943a6b6d6821abbc0ed54 2024-11-21T21:16:47,580 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/ec8dde1ab3a943a6b6d6821abbc0ed54, entries=150, sequenceid=100, filesize=11.7 K 2024-11-21T21:16:47,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/4758f5d824744ae2aeddbc496eccafbc as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/4758f5d824744ae2aeddbc496eccafbc 2024-11-21T21:16:47,584 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/4758f5d824744ae2aeddbc496eccafbc, entries=150, sequenceid=100, filesize=11.7 K 2024-11-21T21:16:47,584 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 5a7f333184706b95cbb8c49fb3d84f31 in 1653ms, sequenceid=100, compaction requested=true 2024-11-21T21:16:47,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5a7f333184706b95cbb8c49fb3d84f31: 2024-11-21T21:16:47,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a7f333184706b95cbb8c49fb3d84f31:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:16:47,584 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:16:47,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:47,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a7f333184706b95cbb8c49fb3d84f31:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:16:47,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:47,584 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:16:47,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a7f333184706b95cbb8c49fb3d84f31:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:16:47,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:16:47,585 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:16:47,585 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): 5a7f333184706b95cbb8c49fb3d84f31/B is initiating minor compaction (all files) 2024-11-21T21:16:47,585 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:16:47,585 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a7f333184706b95cbb8c49fb3d84f31/B in TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:47,585 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): 5a7f333184706b95cbb8c49fb3d84f31/A is initiating minor compaction (all files) 2024-11-21T21:16:47,585 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/9763ad8ab1e34eb9b6f76129cb3c26f1, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/a6a4399ad3a845449361005bacedccb2, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/ec8dde1ab3a943a6b6d6821abbc0ed54] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp, totalSize=35.3 K 2024-11-21T21:16:47,585 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a7f333184706b95cbb8c49fb3d84f31/A in TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:47,585 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/2f5e69d5955e4903a30cf68ea0aafe9d, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/948454736c674488a24016b9fc20517a, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/5d9d2ed2bf3142199f6b50af11c3bf32] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp, totalSize=99.2 K 2024-11-21T21:16:47,585 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:47,585 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. files: [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/2f5e69d5955e4903a30cf68ea0aafe9d, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/948454736c674488a24016b9fc20517a, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/5d9d2ed2bf3142199f6b50af11c3bf32] 2024-11-21T21:16:47,585 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 9763ad8ab1e34eb9b6f76129cb3c26f1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732223801974 2024-11-21T21:16:47,586 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f5e69d5955e4903a30cf68ea0aafe9d, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732223801974 2024-11-21T21:16:47,586 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting a6a4399ad3a845449361005bacedccb2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732223802631 2024-11-21T21:16:47,586 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting ec8dde1ab3a943a6b6d6821abbc0ed54, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732223804788 2024-11-21T21:16:47,586 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 948454736c674488a24016b9fc20517a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732223802631 2024-11-21T21:16:47,586 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d9d2ed2bf3142199f6b50af11c3bf32, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732223804788 2024-11-21T21:16:47,591 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5a7f333184706b95cbb8c49fb3d84f31] 2024-11-21T21:16:47,596 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a7f333184706b95cbb8c49fb3d84f31#B#compaction#500 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:16:47,597 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/ea143fd9542a4ecfb40498b059b03b04 is 50, key is test_row_0/B:col10/1732223804795/Put/seqid=0 2024-11-21T21:16:47,598 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112152339aa16a574d729acf7cfcb827ac43_5a7f333184706b95cbb8c49fb3d84f31 store=[table=TestAcidGuarantees family=A region=5a7f333184706b95cbb8c49fb3d84f31] 2024-11-21T21:16:47,601 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112152339aa16a574d729acf7cfcb827ac43_5a7f333184706b95cbb8c49fb3d84f31, store=[table=TestAcidGuarantees family=A region=5a7f333184706b95cbb8c49fb3d84f31] 2024-11-21T21:16:47,601 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112152339aa16a574d729acf7cfcb827ac43_5a7f333184706b95cbb8c49fb3d84f31 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5a7f333184706b95cbb8c49fb3d84f31] 2024-11-21T21:16:47,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742402_1578 (size=4469) 2024-11-21T21:16:47,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742403_1579 (size=12207) 2024-11-21T21:16:47,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-21T21:16:47,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:47,722 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 5a7f333184706b95cbb8c49fb3d84f31 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-21T21:16:47,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=A 2024-11-21T21:16:47,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:47,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=B 2024-11-21T21:16:47,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:47,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=C 2024-11-21T21:16:47,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:47,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411216bc2ea10132c42d29bc32f20e3fb7fa5_5a7f333184706b95cbb8c49fb3d84f31 is 50, key is test_row_0/A:col10/1732223805957/Put/seqid=0 2024-11-21T21:16:47,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-21T21:16:47,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742404_1580 (size=12154) 2024-11-21T21:16:48,006 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a7f333184706b95cbb8c49fb3d84f31#A#compaction#501 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:16:48,006 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/1650a21e8ae74536ae9e00c65baf77f6 is 175, key is test_row_0/A:col10/1732223804795/Put/seqid=0 2024-11-21T21:16:48,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742405_1581 (size=31161) 2024-11-21T21:16:48,010 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/ea143fd9542a4ecfb40498b059b03b04 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/ea143fd9542a4ecfb40498b059b03b04 2024-11-21T21:16:48,014 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/1650a21e8ae74536ae9e00c65baf77f6 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/1650a21e8ae74536ae9e00c65baf77f6 2024-11-21T21:16:48,018 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a7f333184706b95cbb8c49fb3d84f31/A of 5a7f333184706b95cbb8c49fb3d84f31 into 1650a21e8ae74536ae9e00c65baf77f6(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:16:48,018 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a7f333184706b95cbb8c49fb3d84f31: 2024-11-21T21:16:48,018 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., storeName=5a7f333184706b95cbb8c49fb3d84f31/A, priority=13, startTime=1732223807584; duration=0sec 2024-11-21T21:16:48,018 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:16:48,018 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a7f333184706b95cbb8c49fb3d84f31:A 2024-11-21T21:16:48,018 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:16:48,019 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:16:48,019 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): 5a7f333184706b95cbb8c49fb3d84f31/C is initiating minor compaction (all files) 2024-11-21T21:16:48,019 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a7f333184706b95cbb8c49fb3d84f31/C in TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:48,019 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/c4c0042840664f368aa2766ef2c6cd33, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/17b7277b32c345b0a9563814c5a0e621, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/4758f5d824744ae2aeddbc496eccafbc] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp, totalSize=35.3 K 2024-11-21T21:16:48,020 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4c0042840664f368aa2766ef2c6cd33, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732223801974 2024-11-21T21:16:48,020 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17b7277b32c345b0a9563814c5a0e621, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732223802631 2024-11-21T21:16:48,021 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4758f5d824744ae2aeddbc496eccafbc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732223804788 2024-11-21T21:16:48,021 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a7f333184706b95cbb8c49fb3d84f31/B of 5a7f333184706b95cbb8c49fb3d84f31 into ea143fd9542a4ecfb40498b059b03b04(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:16:48,021 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a7f333184706b95cbb8c49fb3d84f31: 2024-11-21T21:16:48,021 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., storeName=5a7f333184706b95cbb8c49fb3d84f31/B, priority=13, startTime=1732223807584; duration=0sec 2024-11-21T21:16:48,021 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:48,021 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a7f333184706b95cbb8c49fb3d84f31:B 2024-11-21T21:16:48,026 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a7f333184706b95cbb8c49fb3d84f31#C#compaction#503 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:16:48,027 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/455b036b27c84c8cb7656886d1b7c966 is 50, key is test_row_0/C:col10/1732223804795/Put/seqid=0 2024-11-21T21:16:48,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742406_1582 (size=12207) 2024-11-21T21:16:48,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:48,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. as already flushing 2024-11-21T21:16:48,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:48,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223868122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:48,126 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:48,126 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:48,126 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:48,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:48,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223868123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:48,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:48,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:48,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:48,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223868124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:48,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:48,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:48,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223868125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:48,127 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:48,127 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:48,128 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:48,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:48,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:48,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:48,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:48,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42950 deadline: 1732223868126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:48,130 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:48,130 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:48,130 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:48,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:16:48,151 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411216bc2ea10132c42d29bc32f20e3fb7fa5_5a7f333184706b95cbb8c49fb3d84f31 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216bc2ea10132c42d29bc32f20e3fb7fa5_5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:48,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/2f56ccbb652c4c7baca25ebc2c078cca, store: [table=TestAcidGuarantees family=A region=5a7f333184706b95cbb8c49fb3d84f31] 2024-11-21T21:16:48,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/2f56ccbb652c4c7baca25ebc2c078cca is 175, key is test_row_0/A:col10/1732223805957/Put/seqid=0 2024-11-21T21:16:48,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742407_1583 (size=30955) 2024-11-21T21:16:48,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:48,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223868230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:48,232 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:48,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:48,232 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:48,232 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:48,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223868230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:48,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:48,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:48,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:48,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223868231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:48,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:48,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:48,233 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:48,233 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:48,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223868231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:48,233 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:48,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:48,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:48,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:48,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:48,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42950 deadline: 1732223868240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:48,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:48,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:48,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:48,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-21T21:16:48,434 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/455b036b27c84c8cb7656886d1b7c966 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/455b036b27c84c8cb7656886d1b7c966 2024-11-21T21:16:48,437 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a7f333184706b95cbb8c49fb3d84f31/C of 5a7f333184706b95cbb8c49fb3d84f31 into 455b036b27c84c8cb7656886d1b7c966(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:16:48,437 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a7f333184706b95cbb8c49fb3d84f31: 2024-11-21T21:16:48,437 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., storeName=5a7f333184706b95cbb8c49fb3d84f31/C, priority=13, startTime=1732223807584; duration=0sec 2024-11-21T21:16:48,437 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:48,437 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a7f333184706b95cbb8c49fb3d84f31:C 2024-11-21T21:16:48,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:48,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223868440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:48,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:48,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223868441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:48,443 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:48,443 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:48,443 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:48,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:48,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:48,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223868441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:48,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:48,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:48,444 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:48,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:48,444 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:48,444 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:48,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223868441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:48,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:48,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:48,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:48,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:48,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42950 deadline: 1732223868450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:48,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:48,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:48,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:48,556 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/2f56ccbb652c4c7baca25ebc2c078cca 2024-11-21T21:16:48,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/008e499b9d7b438cbc36bfd346f25c03 is 50, key is test_row_0/B:col10/1732223805957/Put/seqid=0 2024-11-21T21:16:48,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742408_1584 (size=12001) 2024-11-21T21:16:48,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:48,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223868750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:48,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:48,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223868750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:48,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:48,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:48,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:48,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:48,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:48,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:48,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223868750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:48,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:48,751 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:48,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:48,751 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:48,751 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:48,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223868750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:48,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:48,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:48,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:48,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:48,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42950 deadline: 1732223868760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:48,762 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:48,762 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:48,762 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:48,965 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/008e499b9d7b438cbc36bfd346f25c03 2024-11-21T21:16:48,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/fe1b07463ef34941b7e591bb193c8551 is 50, key is test_row_0/C:col10/1732223805957/Put/seqid=0 2024-11-21T21:16:48,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742409_1585 (size=12001) 2024-11-21T21:16:49,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-21T21:16:49,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:49,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223869260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:49,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:49,263 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:49,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223869260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:49,263 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:49,263 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:49,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:49,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:49,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:49,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:49,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223869261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:49,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:49,264 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:49,264 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:49,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223869261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:49,264 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:49,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:49,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:49,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:49,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:49,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42950 deadline: 1732223869270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:49,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:49,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:49,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:49,373 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/fe1b07463ef34941b7e591bb193c8551 2024-11-21T21:16:49,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/2f56ccbb652c4c7baca25ebc2c078cca as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/2f56ccbb652c4c7baca25ebc2c078cca 2024-11-21T21:16:49,379 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/2f56ccbb652c4c7baca25ebc2c078cca, entries=150, sequenceid=117, filesize=30.2 K 2024-11-21T21:16:49,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/008e499b9d7b438cbc36bfd346f25c03 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/008e499b9d7b438cbc36bfd346f25c03 2024-11-21T21:16:49,383 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/008e499b9d7b438cbc36bfd346f25c03, entries=150, sequenceid=117, filesize=11.7 K 2024-11-21T21:16:49,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/fe1b07463ef34941b7e591bb193c8551 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/fe1b07463ef34941b7e591bb193c8551 2024-11-21T21:16:49,386 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/fe1b07463ef34941b7e591bb193c8551, entries=150, sequenceid=117, filesize=11.7 K 2024-11-21T21:16:49,386 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 5a7f333184706b95cbb8c49fb3d84f31 in 1664ms, sequenceid=117, compaction requested=false 2024-11-21T21:16:49,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 5a7f333184706b95cbb8c49fb3d84f31: 2024-11-21T21:16:49,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:49,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-21T21:16:49,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-21T21:16:49,389 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-21T21:16:49,389 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2740 sec 2024-11-21T21:16:49,390 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 2.2780 sec 2024-11-21T21:16:50,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:50,272 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5a7f333184706b95cbb8c49fb3d84f31 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-21T21:16:50,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=A 2024-11-21T21:16:50,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:50,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=B 2024-11-21T21:16:50,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:50,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=C 2024-11-21T21:16:50,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:50,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121faae23ed521f46e5b1225875e5738fc5_5a7f333184706b95cbb8c49fb3d84f31 is 50, key is test_row_0/A:col10/1732223810271/Put/seqid=0 2024-11-21T21:16:50,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742410_1586 (size=14794) 2024-11-21T21:16:50,282 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:16:50,284 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121faae23ed521f46e5b1225875e5738fc5_5a7f333184706b95cbb8c49fb3d84f31 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121faae23ed521f46e5b1225875e5738fc5_5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:50,285 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/dce2c58ff0d54c018929ad88963b0015, store: [table=TestAcidGuarantees family=A region=5a7f333184706b95cbb8c49fb3d84f31] 2024-11-21T21:16:50,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/dce2c58ff0d54c018929ad88963b0015 is 175, key is test_row_0/A:col10/1732223810271/Put/seqid=0 2024-11-21T21:16:50,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742411_1587 (size=39749) 2024-11-21T21:16:50,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:50,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223870293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:50,299 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:50,299 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:50,299 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:50,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:50,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223870294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:50,300 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:50,300 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:50,300 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:50,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:50,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223870294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:50,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:50,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:50,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:50,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:50,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42950 deadline: 1732223870295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:50,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:50,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:50,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:50,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:50,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223870296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:50,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:50,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:50,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:50,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:50,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223870410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:50,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:50,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223870411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:50,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:50,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223870411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:50,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:50,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223870411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:50,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:50,415 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:50,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:50,415 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:50,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:50,415 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:50,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:50,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42950 deadline: 1732223870411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:50,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:50,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:50,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:50,415 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:50,415 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:50,415 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:50,421 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:50,422 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:50,422 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:50,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:50,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223870620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:50,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:50,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223870621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:50,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:50,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:50,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:50,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:50,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223870621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:50,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:50,624 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:50,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42950 deadline: 1732223870621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:50,624 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:50,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:50,624 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:50,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:50,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:50,624 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:50,624 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:50,625 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:50,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:50,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223870630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:50,633 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:50,633 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:50,633 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:50,689 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=136, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/dce2c58ff0d54c018929ad88963b0015 2024-11-21T21:16:50,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/be7ea0a0a7a24d17a29a948bca3ed17e is 50, key is test_row_0/B:col10/1732223810271/Put/seqid=0 2024-11-21T21:16:50,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742412_1588 (size=12151) 2024-11-21T21:16:50,700 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/be7ea0a0a7a24d17a29a948bca3ed17e 2024-11-21T21:16:50,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/b71fd2469dd84bdfb8de9667972f61e7 is 50, key is test_row_0/C:col10/1732223810271/Put/seqid=0 2024-11-21T21:16:50,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742413_1589 (size=12151) 2024-11-21T21:16:50,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:50,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223870930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:50,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:50,932 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:50,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223870930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:50,932 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:50,932 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:50,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:50,933 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:50,933 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:50,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223870931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:50,933 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:50,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:50,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:50,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42950 deadline: 1732223870931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:50,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:50,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:50,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:50,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:50,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:50,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:50,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223870940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:50,942 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:50,942 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:50,942 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:51,119 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/b71fd2469dd84bdfb8de9667972f61e7 2024-11-21T21:16:51,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/dce2c58ff0d54c018929ad88963b0015 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/dce2c58ff0d54c018929ad88963b0015 2024-11-21T21:16:51,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/dce2c58ff0d54c018929ad88963b0015, entries=200, sequenceid=136, filesize=38.8 K 2024-11-21T21:16:51,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/be7ea0a0a7a24d17a29a948bca3ed17e as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/be7ea0a0a7a24d17a29a948bca3ed17e 2024-11-21T21:16:51,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/be7ea0a0a7a24d17a29a948bca3ed17e, entries=150, sequenceid=136, filesize=11.9 K 2024-11-21T21:16:51,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/b71fd2469dd84bdfb8de9667972f61e7 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/b71fd2469dd84bdfb8de9667972f61e7 2024-11-21T21:16:51,132 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/b71fd2469dd84bdfb8de9667972f61e7, entries=150, sequenceid=136, filesize=11.9 K 2024-11-21T21:16:51,132 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 5a7f333184706b95cbb8c49fb3d84f31 in 860ms, sequenceid=136, compaction requested=true 2024-11-21T21:16:51,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5a7f333184706b95cbb8c49fb3d84f31: 2024-11-21T21:16:51,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a7f333184706b95cbb8c49fb3d84f31:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:16:51,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:51,133 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:16:51,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a7f333184706b95cbb8c49fb3d84f31:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:16:51,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:51,133 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:16:51,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a7f333184706b95cbb8c49fb3d84f31:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:16:51,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:16:51,134 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:16:51,135 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): 5a7f333184706b95cbb8c49fb3d84f31/B is initiating minor compaction (all files) 2024-11-21T21:16:51,135 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a7f333184706b95cbb8c49fb3d84f31/B in TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:51,135 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/ea143fd9542a4ecfb40498b059b03b04, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/008e499b9d7b438cbc36bfd346f25c03, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/be7ea0a0a7a24d17a29a948bca3ed17e] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp, totalSize=35.5 K 2024-11-21T21:16:51,135 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:16:51,135 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): 5a7f333184706b95cbb8c49fb3d84f31/A is initiating minor compaction (all files) 2024-11-21T21:16:51,135 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a7f333184706b95cbb8c49fb3d84f31/A in TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:51,135 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/1650a21e8ae74536ae9e00c65baf77f6, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/2f56ccbb652c4c7baca25ebc2c078cca, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/dce2c58ff0d54c018929ad88963b0015] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp, totalSize=99.5 K 2024-11-21T21:16:51,135 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:51,135 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. files: [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/1650a21e8ae74536ae9e00c65baf77f6, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/2f56ccbb652c4c7baca25ebc2c078cca, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/dce2c58ff0d54c018929ad88963b0015] 2024-11-21T21:16:51,135 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea143fd9542a4ecfb40498b059b03b04, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732223804788 2024-11-21T21:16:51,135 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 1650a21e8ae74536ae9e00c65baf77f6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732223804788 2024-11-21T21:16:51,136 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 008e499b9d7b438cbc36bfd346f25c03, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732223805954 2024-11-21T21:16:51,136 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f56ccbb652c4c7baca25ebc2c078cca, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732223805954 2024-11-21T21:16:51,136 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting be7ea0a0a7a24d17a29a948bca3ed17e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732223808124 2024-11-21T21:16:51,136 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting dce2c58ff0d54c018929ad88963b0015, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732223808121 2024-11-21T21:16:51,140 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5a7f333184706b95cbb8c49fb3d84f31] 2024-11-21T21:16:51,146 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a7f333184706b95cbb8c49fb3d84f31#B#compaction#510 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:16:51,146 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/b64e473b88854a3b8f8e333eb54a338e is 50, key is test_row_0/B:col10/1732223810271/Put/seqid=0 2024-11-21T21:16:51,149 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121a8036a2a6d294dfbabbbf48e6e9a9e64_5a7f333184706b95cbb8c49fb3d84f31 store=[table=TestAcidGuarantees family=A region=5a7f333184706b95cbb8c49fb3d84f31] 2024-11-21T21:16:51,151 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121a8036a2a6d294dfbabbbf48e6e9a9e64_5a7f333184706b95cbb8c49fb3d84f31, store=[table=TestAcidGuarantees family=A region=5a7f333184706b95cbb8c49fb3d84f31] 2024-11-21T21:16:51,151 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a8036a2a6d294dfbabbbf48e6e9a9e64_5a7f333184706b95cbb8c49fb3d84f31 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5a7f333184706b95cbb8c49fb3d84f31] 2024-11-21T21:16:51,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742414_1590 (size=12459) 2024-11-21T21:16:51,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742415_1591 (size=4469) 2024-11-21T21:16:51,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-21T21:16:51,281 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:16:51,281 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:16:51,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-21T21:16:51,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-21T21:16:51,282 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:16:51,283 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:16:51,283 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:16:51,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-21T21:16:51,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-21T21:16:51,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:51,435 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 5a7f333184706b95cbb8c49fb3d84f31 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-21T21:16:51,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=A 2024-11-21T21:16:51,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:51,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=B 2024-11-21T21:16:51,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:51,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=C 2024-11-21T21:16:51,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:51,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112150468b23a2c6415bb2301197ec60e4c0_5a7f333184706b95cbb8c49fb3d84f31 is 50, key is test_row_0/A:col10/1732223810292/Put/seqid=0 2024-11-21T21:16:51,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742416_1592 (size=12304) 2024-11-21T21:16:51,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:51,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. as already flushing 2024-11-21T21:16:51,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:51,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223871454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:51,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:51,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:51,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:51,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:51,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223871458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:51,463 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:51,463 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:51,463 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:51,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:51,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:51,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223871459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:51,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223871459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:51,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:51,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:51,465 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:51,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:51,465 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:51,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42950 deadline: 1732223871460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:51,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:51,465 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:51,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:51,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:51,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:51,557 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a7f333184706b95cbb8c49fb3d84f31#A#compaction#509 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:16:51,557 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/5ca66206a0a24ce0b06e119430f6456b is 175, key is test_row_0/A:col10/1732223810271/Put/seqid=0 2024-11-21T21:16:51,561 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/b64e473b88854a3b8f8e333eb54a338e as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/b64e473b88854a3b8f8e333eb54a338e 2024-11-21T21:16:51,564 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a7f333184706b95cbb8c49fb3d84f31/B of 5a7f333184706b95cbb8c49fb3d84f31 into b64e473b88854a3b8f8e333eb54a338e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:16:51,564 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a7f333184706b95cbb8c49fb3d84f31: 2024-11-21T21:16:51,564 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., storeName=5a7f333184706b95cbb8c49fb3d84f31/B, priority=13, startTime=1732223811133; duration=0sec 2024-11-21T21:16:51,565 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:16:51,565 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a7f333184706b95cbb8c49fb3d84f31:B 2024-11-21T21:16:51,565 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T21:16:51,566 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T21:16:51,566 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): 5a7f333184706b95cbb8c49fb3d84f31/C is initiating minor compaction (all files) 2024-11-21T21:16:51,566 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a7f333184706b95cbb8c49fb3d84f31/C in TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:51,566 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/455b036b27c84c8cb7656886d1b7c966, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/fe1b07463ef34941b7e591bb193c8551, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/b71fd2469dd84bdfb8de9667972f61e7] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp, totalSize=35.5 K 2024-11-21T21:16:51,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742417_1593 (size=31413) 2024-11-21T21:16:51,567 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 455b036b27c84c8cb7656886d1b7c966, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732223804788 2024-11-21T21:16:51,567 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe1b07463ef34941b7e591bb193c8551, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732223805954 2024-11-21T21:16:51,567 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting b71fd2469dd84bdfb8de9667972f61e7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732223808124 2024-11-21T21:16:51,572 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a7f333184706b95cbb8c49fb3d84f31#C#compaction#512 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:16:51,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:51,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223871571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:51,573 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/cbbe4ce722ca4b8b9e0adab7173e647c is 50, key is test_row_0/C:col10/1732223810271/Put/seqid=0 2024-11-21T21:16:51,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:51,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:51,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:51,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:51,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223871574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:51,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:51,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:51,577 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:51,577 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:51,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223871574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:51,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223871574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:51,577 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:51,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:51,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:51,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42950 deadline: 1732223871574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:51,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:51,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:51,578 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:51,578 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:51,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:51,578 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:51,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:51,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:51,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742418_1594 (size=12459) 2024-11-21T21:16:51,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-21T21:16:51,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:51,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223871780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:51,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:51,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:51,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:51,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:51,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223871780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:51,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:51,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:51,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:51,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:51,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:51,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223871781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:51,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223871780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:51,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:51,785 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:51,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:51,785 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:51,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:51,785 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:51,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:51,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42950 deadline: 1732223871790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:51,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:51,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:51,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:51,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:16:51,846 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112150468b23a2c6415bb2301197ec60e4c0_5a7f333184706b95cbb8c49fb3d84f31 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112150468b23a2c6415bb2301197ec60e4c0_5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:51,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/a9fa2e0497854388afd89b74ba9699fb, store: [table=TestAcidGuarantees family=A region=5a7f333184706b95cbb8c49fb3d84f31] 2024-11-21T21:16:51,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/a9fa2e0497854388afd89b74ba9699fb is 175, key is test_row_0/A:col10/1732223810292/Put/seqid=0 2024-11-21T21:16:51,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742419_1595 (size=31105) 2024-11-21T21:16:51,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-21T21:16:51,970 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/5ca66206a0a24ce0b06e119430f6456b as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/5ca66206a0a24ce0b06e119430f6456b 2024-11-21T21:16:51,973 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a7f333184706b95cbb8c49fb3d84f31/A of 5a7f333184706b95cbb8c49fb3d84f31 into 5ca66206a0a24ce0b06e119430f6456b(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:16:51,973 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a7f333184706b95cbb8c49fb3d84f31: 2024-11-21T21:16:51,973 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., storeName=5a7f333184706b95cbb8c49fb3d84f31/A, priority=13, startTime=1732223811132; duration=0sec 2024-11-21T21:16:51,973 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:51,973 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a7f333184706b95cbb8c49fb3d84f31:A 2024-11-21T21:16:51,983 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/cbbe4ce722ca4b8b9e0adab7173e647c as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/cbbe4ce722ca4b8b9e0adab7173e647c 2024-11-21T21:16:51,986 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5a7f333184706b95cbb8c49fb3d84f31/C of 5a7f333184706b95cbb8c49fb3d84f31 into cbbe4ce722ca4b8b9e0adab7173e647c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:16:51,986 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a7f333184706b95cbb8c49fb3d84f31: 2024-11-21T21:16:51,986 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., storeName=5a7f333184706b95cbb8c49fb3d84f31/C, priority=13, startTime=1732223811133; duration=0sec 2024-11-21T21:16:51,986 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:51,986 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a7f333184706b95cbb8c49fb3d84f31:C 2024-11-21T21:16:52,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:52,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223872090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:52,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:52,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:52,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:52,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223872090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:52,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:52,092 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:52,092 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:52,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:52,092 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:52,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223872090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:52,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:52,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:52,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:52,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:52,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223872090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:52,092 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:52,092 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:52,092 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:52,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:52,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42950 deadline: 1732223872101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:52,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:52,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:52,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:52,250 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=152, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/a9fa2e0497854388afd89b74ba9699fb 2024-11-21T21:16:52,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/054d2e7c581048dba4ac2fd4e35b5f23 is 50, key is test_row_0/B:col10/1732223810292/Put/seqid=0 2024-11-21T21:16:52,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742420_1596 (size=12151) 2024-11-21T21:16:52,265 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/054d2e7c581048dba4ac2fd4e35b5f23 2024-11-21T21:16:52,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/cdd3a7530e204b0ea296aa675b6294d8 is 50, key is test_row_0/C:col10/1732223810292/Put/seqid=0 2024-11-21T21:16:52,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742421_1597 (size=12151) 2024-11-21T21:16:52,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-21T21:16:52,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:52,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223872600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:52,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:52,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:52,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223872601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:52,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:52,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:52,603 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:52,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:52,603 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:52,603 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:52,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223872601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:52,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:52,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:52,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:52,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:52,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223872601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:52,603 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:52,603 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:52,603 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:52,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:52,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42950 deadline: 1732223872610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:52,612 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:52,612 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:52,612 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:52,686 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/cdd3a7530e204b0ea296aa675b6294d8 2024-11-21T21:16:52,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/a9fa2e0497854388afd89b74ba9699fb as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/a9fa2e0497854388afd89b74ba9699fb 2024-11-21T21:16:52,692 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/a9fa2e0497854388afd89b74ba9699fb, entries=150, sequenceid=152, filesize=30.4 K 2024-11-21T21:16:52,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-21T21:16:52,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/054d2e7c581048dba4ac2fd4e35b5f23 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/054d2e7c581048dba4ac2fd4e35b5f23 2024-11-21T21:16:52,696 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/054d2e7c581048dba4ac2fd4e35b5f23, entries=150, sequenceid=152, filesize=11.9 K 2024-11-21T21:16:52,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/cdd3a7530e204b0ea296aa675b6294d8 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/cdd3a7530e204b0ea296aa675b6294d8 2024-11-21T21:16:52,700 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/cdd3a7530e204b0ea296aa675b6294d8, entries=150, sequenceid=152, filesize=11.9 K 2024-11-21T21:16:52,700 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 5a7f333184706b95cbb8c49fb3d84f31 in 1265ms, sequenceid=152, compaction requested=false 2024-11-21T21:16:52,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 5a7f333184706b95cbb8c49fb3d84f31: 2024-11-21T21:16:52,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:52,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-21T21:16:52,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-21T21:16:52,704 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-21T21:16:52,704 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4200 sec 2024-11-21T21:16:52,706 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 1.4230 sec 2024-11-21T21:16:53,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-21T21:16:53,430 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:16:53,430 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:16:53,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-21T21:16:53,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-21T21:16:53,433 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:16:53,433 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:16:53,434 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:16:53,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-21T21:16:53,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-21T21:16:53,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:53,586 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing 5a7f333184706b95cbb8c49fb3d84f31 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-21T21:16:53,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=A 2024-11-21T21:16:53,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:53,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=B 2024-11-21T21:16:53,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:53,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=C 2024-11-21T21:16:53,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:53,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:53,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. as already flushing 2024-11-21T21:16:53,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112152d534af9e7f4081acb1a93169fe6a2c_5a7f333184706b95cbb8c49fb3d84f31 is 50, key is test_row_0/A:col10/1732223811458/Put/seqid=0 2024-11-21T21:16:53,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742422_1598 (size=12304) 2024-11-21T21:16:53,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:16:53,651 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112152d534af9e7f4081acb1a93169fe6a2c_5a7f333184706b95cbb8c49fb3d84f31 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112152d534af9e7f4081acb1a93169fe6a2c_5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:53,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/9c5914a709b74718b344ed628a362fca, store: [table=TestAcidGuarantees family=A region=5a7f333184706b95cbb8c49fb3d84f31] 2024-11-21T21:16:53,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/9c5914a709b74718b344ed628a362fca is 175, key is test_row_0/A:col10/1732223811458/Put/seqid=0 2024-11-21T21:16:53,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742423_1599 (size=31105) 2024-11-21T21:16:53,657 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=171, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/9c5914a709b74718b344ed628a362fca 2024-11-21T21:16:53,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/3f5183e7b33245e98af228f682fcfc2c is 50, key is test_row_0/B:col10/1732223811458/Put/seqid=0 2024-11-21T21:16:53,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742424_1600 (size=12151) 2024-11-21T21:16:53,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:53,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223873677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:53,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:53,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42950 deadline: 1732223873677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:53,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:53,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:53,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:53,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:53,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:53,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223873677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:53,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223873676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:53,687 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:53,687 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:53,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:53,687 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:53,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:53,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:53,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:53,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:53,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:53,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:53,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223873684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:53,691 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:53,691 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:53,691 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:53,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-21T21:16:53,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:53,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223873790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:53,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:53,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:53,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:53,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223873790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:53,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:53,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:53,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:53,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:53,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42950 deadline: 1732223873790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:53,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:53,794 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:53,794 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:53,794 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:53,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:53,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223873790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:53,795 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:53,795 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:53,795 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:53,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:53,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223873800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:53,802 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:53,802 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:53,802 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:54,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:54,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223874000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:54,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:54,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:54,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:54,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:54,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223874000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:54,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:54,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42950 deadline: 1732223874001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:54,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:54,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223874001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:54,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:54,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:54,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:54,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:54,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:54,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:54,006 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:54,006 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:54,006 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:54,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:54,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223874010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:54,014 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:54,014 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:54,014 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:54,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-21T21:16:54,070 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/3f5183e7b33245e98af228f682fcfc2c 2024-11-21T21:16:54,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/60c1c0de92d449b0b8e47b1e2ce6698f is 50, key is test_row_0/C:col10/1732223811458/Put/seqid=0 2024-11-21T21:16:54,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742425_1601 (size=12151) 2024-11-21T21:16:54,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:54,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42950 deadline: 1732223874311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:54,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:54,315 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:54,315 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:54,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223874311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:54,315 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:54,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:54,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:54,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:54,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:54,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223874311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:54,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:54,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:54,316 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:54,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:54,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223874311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:54,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:54,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:54,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:54,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:54,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223874321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:54,326 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:54,326 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:54,326 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:54,484 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/60c1c0de92d449b0b8e47b1e2ce6698f 2024-11-21T21:16:54,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/9c5914a709b74718b344ed628a362fca as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/9c5914a709b74718b344ed628a362fca 2024-11-21T21:16:54,491 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/9c5914a709b74718b344ed628a362fca, entries=150, sequenceid=171, filesize=30.4 K 2024-11-21T21:16:54,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/3f5183e7b33245e98af228f682fcfc2c as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/3f5183e7b33245e98af228f682fcfc2c 2024-11-21T21:16:54,497 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/3f5183e7b33245e98af228f682fcfc2c, entries=150, sequenceid=171, filesize=11.9 K 2024-11-21T21:16:54,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/60c1c0de92d449b0b8e47b1e2ce6698f as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/60c1c0de92d449b0b8e47b1e2ce6698f 2024-11-21T21:16:54,501 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/60c1c0de92d449b0b8e47b1e2ce6698f, entries=150, sequenceid=171, filesize=11.9 K 2024-11-21T21:16:54,502 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 5a7f333184706b95cbb8c49fb3d84f31 in 916ms, sequenceid=171, compaction requested=true 2024-11-21T21:16:54,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for 5a7f333184706b95cbb8c49fb3d84f31: 2024-11-21T21:16:54,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:54,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-21T21:16:54,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-21T21:16:54,506 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-21T21:16:54,506 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0700 sec 2024-11-21T21:16:54,508 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 1.0760 sec 2024-11-21T21:16:54,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-21T21:16:54,571 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:16:54,572 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:16:54,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-21T21:16:54,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-21T21:16:54,574 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:16:54,574 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:16:54,574 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:16:54,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-21T21:16:54,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-21T21:16:54,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:54,727 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing 5a7f333184706b95cbb8c49fb3d84f31 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-21T21:16:54,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=A 2024-11-21T21:16:54,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:54,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=B 2024-11-21T21:16:54,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:54,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=C 2024-11-21T21:16:54,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:54,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411212227f9983b2846238709c579e7a52a64_5a7f333184706b95cbb8c49fb3d84f31 is 50, key is test_row_0/A:col10/1732223813651/Put/seqid=0 2024-11-21T21:16:54,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742426_1602 (size=12304) 2024-11-21T21:16:54,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:16:54,776 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411212227f9983b2846238709c579e7a52a64_5a7f333184706b95cbb8c49fb3d84f31 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212227f9983b2846238709c579e7a52a64_5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:54,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/166e6615572e4ef9bcf107ff526937f5, store: [table=TestAcidGuarantees family=A region=5a7f333184706b95cbb8c49fb3d84f31] 2024-11-21T21:16:54,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/166e6615572e4ef9bcf107ff526937f5 is 175, key is test_row_0/A:col10/1732223813651/Put/seqid=0 2024-11-21T21:16:54,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742427_1603 (size=31105) 2024-11-21T21:16:54,806 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=187, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/166e6615572e4ef9bcf107ff526937f5 2024-11-21T21:16:54,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:54,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. as already flushing 2024-11-21T21:16:54,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/ca71f34441cc4a2fa6f29c50cd4b2a97 is 50, key is test_row_0/B:col10/1732223813651/Put/seqid=0 2024-11-21T21:16:54,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742428_1604 (size=12151) 2024-11-21T21:16:54,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:54,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223874866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:54,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:54,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:54,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223874866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:54,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:54,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:54,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:54,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223874867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:54,875 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:54,875 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:54,875 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:54,875 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:54,875 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:54,875 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:54,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:54,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42950 deadline: 1732223874869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:54,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:54,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:54,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:54,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:54,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223874871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:54,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:54,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:54,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:54,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-21T21:16:54,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:54,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223874980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:54,985 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:54,985 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:54,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:54,985 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:54,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223874980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:54,986 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:54,986 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:54,986 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:54,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:54,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223874980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:54,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:54,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:54,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:54,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:54,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:54,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42950 deadline: 1732223874991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:54,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223874991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:54,998 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:54,998 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:54,998 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:54,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:54,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:54,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:55,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:55,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223875190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:55,194 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:55,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:55,194 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:55,194 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:55,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223875190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:55,194 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:55,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:55,194 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:55,194 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:55,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223875190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:55,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:55,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:55,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:55,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-21T21:16:55,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:55,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223875202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:55,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:55,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:55,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:55,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:55,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42950 deadline: 1732223875210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:55,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:55,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:55,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:55,247 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/ca71f34441cc4a2fa6f29c50cd4b2a97 2024-11-21T21:16:55,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/f28150ce24144b4e9eafb74783169fd1 is 50, key is test_row_0/C:col10/1732223813651/Put/seqid=0 2024-11-21T21:16:55,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742429_1605 (size=12151) 2024-11-21T21:16:55,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:55,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223875501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:55,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:55,506 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:55,506 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:55,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223875501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:55,506 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:55,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:55,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:55,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:55,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:55,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223875504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:55,507 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:55,507 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:55,507 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:55,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:55,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223875511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:55,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:55,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:55,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:55,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:55,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42950 deadline: 1732223875520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:55,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:55,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:55,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:55,678 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/f28150ce24144b4e9eafb74783169fd1 2024-11-21T21:16:55,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/166e6615572e4ef9bcf107ff526937f5 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/166e6615572e4ef9bcf107ff526937f5 2024-11-21T21:16:55,686 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/166e6615572e4ef9bcf107ff526937f5, entries=150, sequenceid=187, filesize=30.4 K 2024-11-21T21:16:55,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/ca71f34441cc4a2fa6f29c50cd4b2a97 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/ca71f34441cc4a2fa6f29c50cd4b2a97 2024-11-21T21:16:55,691 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/ca71f34441cc4a2fa6f29c50cd4b2a97, entries=150, sequenceid=187, filesize=11.9 K 2024-11-21T21:16:55,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/f28150ce24144b4e9eafb74783169fd1 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/f28150ce24144b4e9eafb74783169fd1 2024-11-21T21:16:55,696 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/f28150ce24144b4e9eafb74783169fd1, entries=150, sequenceid=187, filesize=11.9 K 2024-11-21T21:16:55,697 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 5a7f333184706b95cbb8c49fb3d84f31 in 970ms, sequenceid=187, compaction requested=true 2024-11-21T21:16:55,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for 5a7f333184706b95cbb8c49fb3d84f31: 2024-11-21T21:16:55,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:55,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-21T21:16:55,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-21T21:16:55,701 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-21T21:16:55,701 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1250 sec 2024-11-21T21:16:55,705 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 1.1310 sec 2024-11-21T21:16:55,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-21T21:16:55,713 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:16:55,714 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:16:55,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees 2024-11-21T21:16:55,716 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:16:55,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-21T21:16:55,717 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:16:55,717 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:16:55,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-21T21:16:55,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-21T21:16:55,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:55,869 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2902): Flushing 5a7f333184706b95cbb8c49fb3d84f31 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-21T21:16:55,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=A 2024-11-21T21:16:55,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:55,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=B 2024-11-21T21:16:55,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:55,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=C 2024-11-21T21:16:55,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:55,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112153067786bb594d029a4b5a9daba7e27b_5a7f333184706b95cbb8c49fb3d84f31 is 50, key is test_row_0/A:col10/1732223814861/Put/seqid=0 2024-11-21T21:16:55,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742430_1606 (size=12304) 2024-11-21T21:16:55,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:16:55,880 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112153067786bb594d029a4b5a9daba7e27b_5a7f333184706b95cbb8c49fb3d84f31 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112153067786bb594d029a4b5a9daba7e27b_5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:55,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/2fe9ef390d2848b0b071ee97df22eedd, store: [table=TestAcidGuarantees family=A region=5a7f333184706b95cbb8c49fb3d84f31] 2024-11-21T21:16:55,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/2fe9ef390d2848b0b071ee97df22eedd is 175, key is test_row_0/A:col10/1732223814861/Put/seqid=0 2024-11-21T21:16:55,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742431_1607 (size=31105) 2024-11-21T21:16:56,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:56,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. as already flushing 2024-11-21T21:16:56,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-21T21:16:56,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:56,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223876027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:56,032 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:56,032 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:56,032 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:56,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:56,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223876028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:56,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:56,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:56,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:56,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:56,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223876029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:56,034 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:56,034 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:56,034 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:56,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:56,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223876030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:56,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:56,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:56,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:56,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:56,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42950 deadline: 1732223876030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:56,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:56,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:56,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:56,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:56,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223876141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:56,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:56,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223876141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:56,144 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:56,144 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:56,144 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:56,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:56,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:56,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:56,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223876141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:56,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:56,144 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:56,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:56,144 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:56,144 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:56,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223876141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:56,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:56,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:56,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:56,286 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=203, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/2fe9ef390d2848b0b071ee97df22eedd 2024-11-21T21:16:56,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/a560ff0448fe4de2be7d16a993c6ef99 is 50, key is test_row_0/B:col10/1732223814861/Put/seqid=0 2024-11-21T21:16:56,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742432_1608 (size=12151) 2024-11-21T21:16:56,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-21T21:16:56,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:56,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223876351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:56,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:56,353 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:56,353 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:56,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223876351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:56,353 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:56,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:56,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:56,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:56,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:56,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223876351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:56,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:56,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:56,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:56,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:56,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223876351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:56,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:56,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:56,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:56,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:56,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223876660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:56,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:56,665 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:56,665 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:56,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223876660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:56,665 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:56,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:56,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223876660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:56,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:56,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:56,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:56,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:56,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223876660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:56,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:56,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:56,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:56,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:56,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:56,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:56,696 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/a560ff0448fe4de2be7d16a993c6ef99 2024-11-21T21:16:56,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/842827bcbe9a4cb3b3294c5ff83c72c1 is 50, key is test_row_0/C:col10/1732223814861/Put/seqid=0 2024-11-21T21:16:56,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742433_1609 (size=12151) 2024-11-21T21:16:56,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-21T21:16:57,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:57,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42950 deadline: 1732223877050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:57,053 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,41213,1732223653469 for 'test_row_1' in 5a7f333184706b95cbb8c49fb3d84f31 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:16:57,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:57,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:57,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:57,114 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/842827bcbe9a4cb3b3294c5ff83c72c1 2024-11-21T21:16:57,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/2fe9ef390d2848b0b071ee97df22eedd as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/2fe9ef390d2848b0b071ee97df22eedd 2024-11-21T21:16:57,121 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/2fe9ef390d2848b0b071ee97df22eedd, entries=150, sequenceid=203, filesize=30.4 K 2024-11-21T21:16:57,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/a560ff0448fe4de2be7d16a993c6ef99 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/a560ff0448fe4de2be7d16a993c6ef99 2024-11-21T21:16:57,124 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/a560ff0448fe4de2be7d16a993c6ef99, entries=150, sequenceid=203, filesize=11.9 K 2024-11-21T21:16:57,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/842827bcbe9a4cb3b3294c5ff83c72c1 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/842827bcbe9a4cb3b3294c5ff83c72c1 2024-11-21T21:16:57,128 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/842827bcbe9a4cb3b3294c5ff83c72c1, entries=150, sequenceid=203, filesize=11.9 K 2024-11-21T21:16:57,129 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 5a7f333184706b95cbb8c49fb3d84f31 in 1260ms, sequenceid=203, compaction requested=true 2024-11-21T21:16:57,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2603): Flush status journal for 5a7f333184706b95cbb8c49fb3d84f31: 2024-11-21T21:16:57,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:57,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=171 2024-11-21T21:16:57,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=171 2024-11-21T21:16:57,132 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-21T21:16:57,132 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4140 sec 2024-11-21T21:16:57,134 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees in 1.4190 sec 2024-11-21T21:16:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:57,173 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5a7f333184706b95cbb8c49fb3d84f31 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-21T21:16:57,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=A 2024-11-21T21:16:57,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:57,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=B 2024-11-21T21:16:57,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:57,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=C 2024-11-21T21:16:57,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:57,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411214c325f35a3fe47df9172a8701f408d40_5a7f333184706b95cbb8c49fb3d84f31 is 50, key is test_row_0/A:col10/1732223816027/Put/seqid=0 2024-11-21T21:16:57,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742434_1610 (size=14794) 2024-11-21T21:16:57,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:57,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223877195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:57,203 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:57,203 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:57,203 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:57,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:57,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223877196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:57,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:57,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:57,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:57,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:57,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223877196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:57,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:57,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:57,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:57,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:57,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223877198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:57,204 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:57,205 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:57,205 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:57,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:57,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223877310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:57,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:57,312 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:57,313 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223877311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:57,313 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:57,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:57,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:57,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:57,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223877311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:57,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:57,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:57,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223877311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:57,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:57,313 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:57,313 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:57,313 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:57,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:57,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223877521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:57,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:57,527 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:57,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:57,527 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:57,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223877521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:57,527 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:57,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223877521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:57,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:57,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223877522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:57,528 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:57,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:57,528 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:57,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:57,528 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:57,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:57,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:57,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:57,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:57,585 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:16:57,588 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411214c325f35a3fe47df9172a8701f408d40_5a7f333184706b95cbb8c49fb3d84f31 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214c325f35a3fe47df9172a8701f408d40_5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:57,589 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/d161e76da3334125bcf42dff9dff516b, store: [table=TestAcidGuarantees family=A region=5a7f333184706b95cbb8c49fb3d84f31] 2024-11-21T21:16:57,589 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/d161e76da3334125bcf42dff9dff516b is 175, key is test_row_0/A:col10/1732223816027/Put/seqid=0 2024-11-21T21:16:57,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742435_1611 (size=39749) 2024-11-21T21:16:57,600 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=220, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/d161e76da3334125bcf42dff9dff516b 2024-11-21T21:16:57,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/c4a80005f623433cba6110dd4e86b35e is 50, key is test_row_0/B:col10/1732223816027/Put/seqid=0 2024-11-21T21:16:57,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742436_1612 (size=12151) 2024-11-21T21:16:57,631 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/c4a80005f623433cba6110dd4e86b35e 2024-11-21T21:16:57,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/33b44a57b06c4b68b8ecece66fc375c1 is 50, key is test_row_0/C:col10/1732223816027/Put/seqid=0 2024-11-21T21:16:57,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742437_1613 (size=12151) 2024-11-21T21:16:57,652 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/33b44a57b06c4b68b8ecece66fc375c1 2024-11-21T21:16:57,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/d161e76da3334125bcf42dff9dff516b as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/d161e76da3334125bcf42dff9dff516b 2024-11-21T21:16:57,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/d161e76da3334125bcf42dff9dff516b, entries=200, sequenceid=220, filesize=38.8 K 2024-11-21T21:16:57,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/c4a80005f623433cba6110dd4e86b35e as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/c4a80005f623433cba6110dd4e86b35e 2024-11-21T21:16:57,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/c4a80005f623433cba6110dd4e86b35e, entries=150, sequenceid=220, filesize=11.9 K 2024-11-21T21:16:57,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/33b44a57b06c4b68b8ecece66fc375c1 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/33b44a57b06c4b68b8ecece66fc375c1 2024-11-21T21:16:57,666 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/33b44a57b06c4b68b8ecece66fc375c1, entries=150, sequenceid=220, filesize=11.9 K 2024-11-21T21:16:57,666 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 5a7f333184706b95cbb8c49fb3d84f31 in 493ms, sequenceid=220, compaction requested=true 2024-11-21T21:16:57,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5a7f333184706b95cbb8c49fb3d84f31: 2024-11-21T21:16:57,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a7f333184706b95cbb8c49fb3d84f31:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:16:57,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:57,667 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-21T21:16:57,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a7f333184706b95cbb8c49fb3d84f31:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:16:57,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:57,667 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-21T21:16:57,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a7f333184706b95cbb8c49fb3d84f31:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:16:57,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:16:57,669 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 73214 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-21T21:16:57,669 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): 5a7f333184706b95cbb8c49fb3d84f31/B is initiating minor compaction (all files) 2024-11-21T21:16:57,669 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a7f333184706b95cbb8c49fb3d84f31/B in TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:57,669 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/b64e473b88854a3b8f8e333eb54a338e, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/054d2e7c581048dba4ac2fd4e35b5f23, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/3f5183e7b33245e98af228f682fcfc2c, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/ca71f34441cc4a2fa6f29c50cd4b2a97, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/a560ff0448fe4de2be7d16a993c6ef99, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/c4a80005f623433cba6110dd4e86b35e] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp, totalSize=71.5 K 2024-11-21T21:16:57,670 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting b64e473b88854a3b8f8e333eb54a338e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732223808124 2024-11-21T21:16:57,670 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 195582 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-21T21:16:57,670 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): 5a7f333184706b95cbb8c49fb3d84f31/A is initiating minor compaction (all files) 2024-11-21T21:16:57,670 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a7f333184706b95cbb8c49fb3d84f31/A in TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:57,670 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/5ca66206a0a24ce0b06e119430f6456b, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/a9fa2e0497854388afd89b74ba9699fb, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/9c5914a709b74718b344ed628a362fca, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/166e6615572e4ef9bcf107ff526937f5, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/2fe9ef390d2848b0b071ee97df22eedd, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/d161e76da3334125bcf42dff9dff516b] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp, totalSize=191.0 K 2024-11-21T21:16:57,670 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=10 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:57,670 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. files: [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/5ca66206a0a24ce0b06e119430f6456b, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/a9fa2e0497854388afd89b74ba9699fb, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/9c5914a709b74718b344ed628a362fca, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/166e6615572e4ef9bcf107ff526937f5, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/2fe9ef390d2848b0b071ee97df22eedd, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/d161e76da3334125bcf42dff9dff516b] 2024-11-21T21:16:57,670 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 054d2e7c581048dba4ac2fd4e35b5f23, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732223810292 2024-11-21T21:16:57,671 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ca66206a0a24ce0b06e119430f6456b, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732223808124 2024-11-21T21:16:57,671 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f5183e7b33245e98af228f682fcfc2c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732223811457 2024-11-21T21:16:57,671 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting ca71f34441cc4a2fa6f29c50cd4b2a97, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732223813634 2024-11-21T21:16:57,671 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9fa2e0497854388afd89b74ba9699fb, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732223810292 2024-11-21T21:16:57,671 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c5914a709b74718b344ed628a362fca, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732223811457 2024-11-21T21:16:57,671 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting a560ff0448fe4de2be7d16a993c6ef99, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732223814861 2024-11-21T21:16:57,671 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting c4a80005f623433cba6110dd4e86b35e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732223816027 2024-11-21T21:16:57,671 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 166e6615572e4ef9bcf107ff526937f5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732223813634 2024-11-21T21:16:57,672 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fe9ef390d2848b0b071ee97df22eedd, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732223814861 2024-11-21T21:16:57,673 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting d161e76da3334125bcf42dff9dff516b, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732223816023 2024-11-21T21:16:57,683 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a7f333184706b95cbb8c49fb3d84f31#B#compaction#527 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:16:57,684 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/3bbb1c80e2bc422c8f58d1b4e8035c0e is 50, key is test_row_0/B:col10/1732223816027/Put/seqid=0 2024-11-21T21:16:57,685 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5a7f333184706b95cbb8c49fb3d84f31] 2024-11-21T21:16:57,686 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121756f5d7c45db45c69090435c28cd984b_5a7f333184706b95cbb8c49fb3d84f31 store=[table=TestAcidGuarantees family=A region=5a7f333184706b95cbb8c49fb3d84f31] 2024-11-21T21:16:57,690 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121756f5d7c45db45c69090435c28cd984b_5a7f333184706b95cbb8c49fb3d84f31, store=[table=TestAcidGuarantees family=A region=5a7f333184706b95cbb8c49fb3d84f31] 2024-11-21T21:16:57,690 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121756f5d7c45db45c69090435c28cd984b_5a7f333184706b95cbb8c49fb3d84f31 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5a7f333184706b95cbb8c49fb3d84f31] 2024-11-21T21:16:57,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742438_1614 (size=12663) 2024-11-21T21:16:57,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742439_1615 (size=4469) 2024-11-21T21:16:57,703 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a7f333184706b95cbb8c49fb3d84f31#A#compaction#528 average throughput is 1.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:16:57,704 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/b3d2f126ca9e426aa9f07e55b8169554 is 175, key is test_row_0/A:col10/1732223816027/Put/seqid=0 2024-11-21T21:16:57,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742440_1616 (size=31617) 2024-11-21T21:16:57,717 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/b3d2f126ca9e426aa9f07e55b8169554 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/b3d2f126ca9e426aa9f07e55b8169554 2024-11-21T21:16:57,723 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 5a7f333184706b95cbb8c49fb3d84f31/A of 5a7f333184706b95cbb8c49fb3d84f31 into b3d2f126ca9e426aa9f07e55b8169554(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:16:57,723 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a7f333184706b95cbb8c49fb3d84f31: 2024-11-21T21:16:57,723 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., storeName=5a7f333184706b95cbb8c49fb3d84f31/A, priority=10, startTime=1732223817666; duration=0sec 2024-11-21T21:16:57,724 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:16:57,724 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a7f333184706b95cbb8c49fb3d84f31:A 2024-11-21T21:16:57,724 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-21T21:16:57,725 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 73214 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-21T21:16:57,725 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): 5a7f333184706b95cbb8c49fb3d84f31/C is initiating minor compaction (all files) 2024-11-21T21:16:57,725 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a7f333184706b95cbb8c49fb3d84f31/C in TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:57,725 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/cbbe4ce722ca4b8b9e0adab7173e647c, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/cdd3a7530e204b0ea296aa675b6294d8, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/60c1c0de92d449b0b8e47b1e2ce6698f, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/f28150ce24144b4e9eafb74783169fd1, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/842827bcbe9a4cb3b3294c5ff83c72c1, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/33b44a57b06c4b68b8ecece66fc375c1] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp, totalSize=71.5 K 2024-11-21T21:16:57,725 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbbe4ce722ca4b8b9e0adab7173e647c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732223808124 2024-11-21T21:16:57,726 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdd3a7530e204b0ea296aa675b6294d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732223810292 2024-11-21T21:16:57,726 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60c1c0de92d449b0b8e47b1e2ce6698f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732223811457 2024-11-21T21:16:57,726 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting f28150ce24144b4e9eafb74783169fd1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732223813634 2024-11-21T21:16:57,726 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 842827bcbe9a4cb3b3294c5ff83c72c1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732223814861 2024-11-21T21:16:57,726 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33b44a57b06c4b68b8ecece66fc375c1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732223816027 2024-11-21T21:16:57,738 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a7f333184706b95cbb8c49fb3d84f31#C#compaction#529 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:16:57,738 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/1703c1312c72449390654bf2b39c8a18 is 50, key is test_row_0/C:col10/1732223816027/Put/seqid=0 2024-11-21T21:16:57,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742441_1617 (size=12663) 2024-11-21T21:16:57,746 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/1703c1312c72449390654bf2b39c8a18 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/1703c1312c72449390654bf2b39c8a18 2024-11-21T21:16:57,750 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 5a7f333184706b95cbb8c49fb3d84f31/C of 5a7f333184706b95cbb8c49fb3d84f31 into 1703c1312c72449390654bf2b39c8a18(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:16:57,750 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a7f333184706b95cbb8c49fb3d84f31: 2024-11-21T21:16:57,750 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., storeName=5a7f333184706b95cbb8c49fb3d84f31/C, priority=10, startTime=1732223817667; duration=0sec 2024-11-21T21:16:57,750 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:57,750 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a7f333184706b95cbb8c49fb3d84f31:C 2024-11-21T21:16:57,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:57,833 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5a7f333184706b95cbb8c49fb3d84f31 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-21T21:16:57,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=A 2024-11-21T21:16:57,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:57,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=B 2024-11-21T21:16:57,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:57,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=C 2024-11-21T21:16:57,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:57,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411216323518ff31a475191ff308d40cb0e62_5a7f333184706b95cbb8c49fb3d84f31 is 50, key is test_row_0/A:col10/1732223817196/Put/seqid=0 2024-11-21T21:16:57,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742442_1618 (size=17284) 2024-11-21T21:16:57,844 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:16:57,850 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411216323518ff31a475191ff308d40cb0e62_5a7f333184706b95cbb8c49fb3d84f31 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216323518ff31a475191ff308d40cb0e62_5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:57,851 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/1ed8f3988ef54f8287e273b0d1d1e037, store: [table=TestAcidGuarantees family=A region=5a7f333184706b95cbb8c49fb3d84f31] 2024-11-21T21:16:57,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/1ed8f3988ef54f8287e273b0d1d1e037 is 175, key is test_row_0/A:col10/1732223817196/Put/seqid=0 2024-11-21T21:16:57,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-21T21:16:57,861 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:16:57,861 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:16:57,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees 2024-11-21T21:16:57,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-21T21:16:57,864 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:16:57,864 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:16:57,865 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:16:57,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742443_1619 (size=48389) 2024-11-21T21:16:57,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:57,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223877857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:57,894 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:57,894 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:57,894 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:57,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:57,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223877894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:57,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:57,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223877894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:57,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:57,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:57,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:57,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:57,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223877894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:57,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:57,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:57,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:57,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:57,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:57,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:57,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-21T21:16:58,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:58,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223878000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:58,003 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:58,003 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:58,003 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:58,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:58,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223878010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:58,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:58,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:58,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223878010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:58,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:58,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:58,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:58,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:58,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:58,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:58,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223878010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:58,014 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:58,014 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:58,014 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:58,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-21T21:16:58,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:58,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. as already flushing 2024-11-21T21:16:58,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:58,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:58,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:58,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:58,106 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/3bbb1c80e2bc422c8f58d1b4e8035c0e as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/3bbb1c80e2bc422c8f58d1b4e8035c0e 2024-11-21T21:16:58,111 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 5a7f333184706b95cbb8c49fb3d84f31/B of 5a7f333184706b95cbb8c49fb3d84f31 into 3bbb1c80e2bc422c8f58d1b4e8035c0e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:16:58,111 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a7f333184706b95cbb8c49fb3d84f31: 2024-11-21T21:16:58,111 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., storeName=5a7f333184706b95cbb8c49fb3d84f31/B, priority=10, startTime=1732223817667; duration=0sec 2024-11-21T21:16:58,111 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:16:58,111 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a7f333184706b95cbb8c49fb3d84f31:B 2024-11-21T21:16:58,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-21T21:16:58,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:58,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. as already flushing 2024-11-21T21:16:58,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:58,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:58,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:58,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:58,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-21T21:16:58,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:58,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223878210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:58,213 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:58,213 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:58,213 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:58,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:58,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223878221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:58,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:58,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:58,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:58,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223878221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:58,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:58,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:58,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:58,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:58,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:58,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223878229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:58,231 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:58,231 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:58,231 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:58,276 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=239, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/1ed8f3988ef54f8287e273b0d1d1e037 2024-11-21T21:16:58,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/b3205ffaa342496eb7c6911e368b537e is 50, key is test_row_0/B:col10/1732223817196/Put/seqid=0 2024-11-21T21:16:58,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742444_1620 (size=12151) 2024-11-21T21:16:58,312 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/b3205ffaa342496eb7c6911e368b537e 2024-11-21T21:16:58,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-21T21:16:58,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:58,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. as already flushing 2024-11-21T21:16:58,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:58,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:58,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:58,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:16:58,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/f916b2825cb342e6966d4f10e84177fa is 50, key is test_row_0/C:col10/1732223817196/Put/seqid=0 2024-11-21T21:16:58,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742445_1621 (size=12151) 2024-11-21T21:16:58,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/f916b2825cb342e6966d4f10e84177fa 2024-11-21T21:16:58,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/1ed8f3988ef54f8287e273b0d1d1e037 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/1ed8f3988ef54f8287e273b0d1d1e037 2024-11-21T21:16:58,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/1ed8f3988ef54f8287e273b0d1d1e037, entries=250, sequenceid=239, filesize=47.3 K 2024-11-21T21:16:58,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/b3205ffaa342496eb7c6911e368b537e as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/b3205ffaa342496eb7c6911e368b537e 2024-11-21T21:16:58,365 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/b3205ffaa342496eb7c6911e368b537e, entries=150, sequenceid=239, filesize=11.9 K 2024-11-21T21:16:58,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/f916b2825cb342e6966d4f10e84177fa as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/f916b2825cb342e6966d4f10e84177fa 2024-11-21T21:16:58,372 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/f916b2825cb342e6966d4f10e84177fa, entries=150, sequenceid=239, filesize=11.9 K 2024-11-21T21:16:58,376 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=93.93 KB/96180 for 5a7f333184706b95cbb8c49fb3d84f31 in 543ms, sequenceid=239, compaction requested=false 2024-11-21T21:16:58,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5a7f333184706b95cbb8c49fb3d84f31: 2024-11-21T21:16:58,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-21T21:16:58,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:58,477 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2902): Flushing 5a7f333184706b95cbb8c49fb3d84f31 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-21T21:16:58,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=A 2024-11-21T21:16:58,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:58,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=B 2024-11-21T21:16:58,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:58,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=C 2024-11-21T21:16:58,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:58,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-21T21:16:58,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b54e652040c94d21b34bbebdcb4b5c47_5a7f333184706b95cbb8c49fb3d84f31 is 50, key is test_row_0/A:col10/1732223817856/Put/seqid=0 2024-11-21T21:16:58,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742446_1622 (size=12304) 2024-11-21T21:16:58,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:58,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. as already flushing 2024-11-21T21:16:58,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:58,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223878545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:58,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:58,550 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:58,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223878546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:58,550 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:58,551 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:58,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:58,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:58,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:58,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:58,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223878547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:58,553 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:58,553 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:58,553 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:58,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:58,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223878548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:58,553 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:58,553 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:58,553 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:58,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:58,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223878660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:58,663 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:58,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:58,663 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:58,663 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:58,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223878661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:58,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:58,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:58,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:58,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:58,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223878661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:58,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:58,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:58,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:58,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:58,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223878661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:58,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:58,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:58,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:58,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:58,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223878870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:58,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:58,873 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:58,873 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:58,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223878870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:58,873 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:58,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:58,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:58,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:58,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:58,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223878871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:58,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:58,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:58,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:58,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:58,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223878871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:58,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:58,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:58,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:58,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:16:58,917 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b54e652040c94d21b34bbebdcb4b5c47_5a7f333184706b95cbb8c49fb3d84f31 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b54e652040c94d21b34bbebdcb4b5c47_5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:58,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/f932256da73d4bbfb70bbc355f29515a, store: [table=TestAcidGuarantees family=A region=5a7f333184706b95cbb8c49fb3d84f31] 2024-11-21T21:16:58,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/f932256da73d4bbfb70bbc355f29515a is 175, key is test_row_0/A:col10/1732223817856/Put/seqid=0 2024-11-21T21:16:58,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742447_1623 (size=31105) 2024-11-21T21:16:58,922 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=257, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/f932256da73d4bbfb70bbc355f29515a 2024-11-21T21:16:58,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/2a9f1c94433f4068b5b9911943a67636 is 50, key is test_row_0/B:col10/1732223817856/Put/seqid=0 2024-11-21T21:16:58,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742448_1624 (size=12151) 2024-11-21T21:16:59,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-21T21:16:59,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:59,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42950 deadline: 1732223879080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:59,085 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 849dee24a242,41213,1732223653469 for 'test_row_1' in 5a7f333184706b95cbb8c49fb3d84f31 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T21:16:59,085 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:59,085 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:59,085 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:59,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:59,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223879180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:59,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:59,184 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:59,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223879180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:59,184 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:59,184 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:59,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:59,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:59,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:59,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:59,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223879181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:59,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:59,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:59,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:59,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:59,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223879181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:59,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:59,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:59,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:59,331 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/2a9f1c94433f4068b5b9911943a67636 2024-11-21T21:16:59,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/ce33ebb2c40b4ec3a691e17a69cbf713 is 50, key is test_row_0/C:col10/1732223817856/Put/seqid=0 2024-11-21T21:16:59,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742449_1625 (size=12151) 2024-11-21T21:16:59,349 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/ce33ebb2c40b4ec3a691e17a69cbf713 2024-11-21T21:16:59,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/f932256da73d4bbfb70bbc355f29515a as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/f932256da73d4bbfb70bbc355f29515a 2024-11-21T21:16:59,360 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/f932256da73d4bbfb70bbc355f29515a, entries=150, sequenceid=257, filesize=30.4 K 2024-11-21T21:16:59,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/2a9f1c94433f4068b5b9911943a67636 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/2a9f1c94433f4068b5b9911943a67636 2024-11-21T21:16:59,366 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/2a9f1c94433f4068b5b9911943a67636, entries=150, sequenceid=257, filesize=11.9 K 2024-11-21T21:16:59,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/ce33ebb2c40b4ec3a691e17a69cbf713 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/ce33ebb2c40b4ec3a691e17a69cbf713 2024-11-21T21:16:59,371 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/ce33ebb2c40b4ec3a691e17a69cbf713, entries=150, sequenceid=257, filesize=11.9 K 2024-11-21T21:16:59,372 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 5a7f333184706b95cbb8c49fb3d84f31 in 894ms, sequenceid=257, compaction requested=true 2024-11-21T21:16:59,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2603): Flush status journal for 5a7f333184706b95cbb8c49fb3d84f31: 2024-11-21T21:16:59,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:16:59,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=173 2024-11-21T21:16:59,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=173 2024-11-21T21:16:59,376 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-21T21:16:59,376 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5090 sec 2024-11-21T21:16:59,378 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees in 1.5150 sec 2024-11-21T21:16:59,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:16:59,693 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5a7f333184706b95cbb8c49fb3d84f31 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-21T21:16:59,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=A 2024-11-21T21:16:59,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:59,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=B 2024-11-21T21:16:59,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:59,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=C 2024-11-21T21:16:59,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:16:59,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a247247819e442e3b2719fe140e33b91_5a7f333184706b95cbb8c49fb3d84f31 is 50, key is test_row_0/A:col10/1732223818545/Put/seqid=0 2024-11-21T21:16:59,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742450_1626 (size=12454) 2024-11-21T21:16:59,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:59,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223879745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:59,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:59,751 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:59,751 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:59,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223879746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:59,751 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:59,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:59,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:59,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223879747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:59,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:59,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:59,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:59,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:59,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:59,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:59,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223879748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:59,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:59,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:59,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:59,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:59,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223879860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:59,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:59,864 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:59,864 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:59,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223879860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:59,864 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:59,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:59,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:59,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:59,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:59,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223879860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:59,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:59,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:59,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:16:59,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:16:59,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223879861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:16:59,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:59,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:16:59,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:17:00,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-21T21:17:00,020 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:17:00,021 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T21:17:00,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=174, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees 2024-11-21T21:17:00,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-21T21:17:00,022 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=174, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T21:17:00,023 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=174, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T21:17:00,023 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=175, ppid=174, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T21:17:00,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:17:00,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223880070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:17:00,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:17:00,074 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:17:00,074 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:17:00,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223880071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:17:00,074 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:17:00,074 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:17:00,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:17:00,074 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:17:00,074 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:17:00,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223880071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:17:00,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:17:00,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:17:00,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:17:00,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:17:00,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223880071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:17:00,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:17:00,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:17:00,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:17:00,114 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:17:00,117 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a247247819e442e3b2719fe140e33b91_5a7f333184706b95cbb8c49fb3d84f31 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a247247819e442e3b2719fe140e33b91_5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:17:00,118 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/f58f91a6c9c04086ab52723f8275480c, store: [table=TestAcidGuarantees family=A region=5a7f333184706b95cbb8c49fb3d84f31] 2024-11-21T21:17:00,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/f58f91a6c9c04086ab52723f8275480c is 175, key is test_row_0/A:col10/1732223818545/Put/seqid=0 2024-11-21T21:17:00,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742451_1627 (size=31255) 2024-11-21T21:17:00,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-21T21:17:00,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-21T21:17:00,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:17:00,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. as already flushing 2024-11-21T21:17:00,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:17:00,175 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:17:00,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:17:00,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:17:00,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-21T21:17:00,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:17:00,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. as already flushing 2024-11-21T21:17:00,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:17:00,328 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:17:00,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:17:00,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:17:00,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-21T21:17:00,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:17:00,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223880380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:17:00,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:17:00,384 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:17:00,384 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:17:00,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223880381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:17:00,384 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:17:00,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:17:00,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:17:00,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:17:00,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:17:00,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223880381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:17:00,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:17:00,385 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:17:00,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223880381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:17:00,385 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:17:00,385 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:17:00,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:17:00,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:17:00,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:17:00,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-21T21:17:00,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:17:00,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. as already flushing 2024-11-21T21:17:00,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:17:00,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:17:00,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:17:00,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:17:00,528 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=273, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/f58f91a6c9c04086ab52723f8275480c 2024-11-21T21:17:00,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/9b3e628cb0e4438e9092a6b86ae64ba5 is 50, key is test_row_0/B:col10/1732223818545/Put/seqid=0 2024-11-21T21:17:00,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742452_1628 (size=12301) 2024-11-21T21:17:00,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-21T21:17:00,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:17:00,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. as already flushing 2024-11-21T21:17:00,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:17:00,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:17:00,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:17:00,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:17:00,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-21T21:17:00,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-21T21:17:00,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:17:00,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. as already flushing 2024-11-21T21:17:00,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:17:00,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:17:00,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:17:00,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:17:00,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:17:00,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42984 deadline: 1732223880891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:17:00,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:17:00,893 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:17:00,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42970 deadline: 1732223880891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:17:00,893 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:17:00,893 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:17:00,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:17:00,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:17:00,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:17:00,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42986 deadline: 1732223880891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:17:00,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:17:00,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T21:17:00,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:17:00,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42942 deadline: 1732223880891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 2024-11-21T21:17:00,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:17:00,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:17:00,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 , the old value is region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:17:00,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T21:17:00,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., hostname=849dee24a242,41213,1732223653469, seqNum=5 because the exception is null or not the one we care about 2024-11-21T21:17:00,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-21T21:17:00,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:17:00,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. as already flushing 2024-11-21T21:17:00,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:17:00,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:17:00,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:17:00,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:17:00,974 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/9b3e628cb0e4438e9092a6b86ae64ba5 2024-11-21T21:17:00,981 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/ce38058bcb404ab38b22f9d45e2ef8ee is 50, key is test_row_0/C:col10/1732223818545/Put/seqid=0 2024-11-21T21:17:01,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742453_1629 (size=12301) 2024-11-21T21:17:01,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-21T21:17:01,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:17:01,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. as already flushing 2024-11-21T21:17:01,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:17:01,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:17:01,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:17:01,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:17:01,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-21T21:17:01,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-21T21:17:01,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:17:01,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. as already flushing 2024-11-21T21:17:01,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:17:01,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:17:01,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:17:01,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:17:01,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-21T21:17:01,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:17:01,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. as already flushing 2024-11-21T21:17:01,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:17:01,398 ERROR [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:17:01,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:17:01,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T21:17:01,403 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/ce38058bcb404ab38b22f9d45e2ef8ee 2024-11-21T21:17:01,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/f58f91a6c9c04086ab52723f8275480c as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/f58f91a6c9c04086ab52723f8275480c 2024-11-21T21:17:01,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/f58f91a6c9c04086ab52723f8275480c, entries=150, sequenceid=273, filesize=30.5 K 2024-11-21T21:17:01,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/9b3e628cb0e4438e9092a6b86ae64ba5 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/9b3e628cb0e4438e9092a6b86ae64ba5 2024-11-21T21:17:01,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/9b3e628cb0e4438e9092a6b86ae64ba5, entries=150, sequenceid=273, filesize=12.0 K 2024-11-21T21:17:01,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/ce38058bcb404ab38b22f9d45e2ef8ee as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/ce38058bcb404ab38b22f9d45e2ef8ee 2024-11-21T21:17:01,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/ce38058bcb404ab38b22f9d45e2ef8ee, entries=150, sequenceid=273, filesize=12.0 K 2024-11-21T21:17:01,416 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 5a7f333184706b95cbb8c49fb3d84f31 in 1723ms, sequenceid=273, compaction requested=true 2024-11-21T21:17:01,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5a7f333184706b95cbb8c49fb3d84f31: 2024-11-21T21:17:01,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a7f333184706b95cbb8c49fb3d84f31:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T21:17:01,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:17:01,416 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:17:01,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a7f333184706b95cbb8c49fb3d84f31:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T21:17:01,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:17:01,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5a7f333184706b95cbb8c49fb3d84f31:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T21:17:01,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:17:01,416 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:17:01,417 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142366 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:17:01,417 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1541): 5a7f333184706b95cbb8c49fb3d84f31/A is initiating minor compaction (all files) 2024-11-21T21:17:01,417 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a7f333184706b95cbb8c49fb3d84f31/A in TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:17:01,417 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/b3d2f126ca9e426aa9f07e55b8169554, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/1ed8f3988ef54f8287e273b0d1d1e037, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/f932256da73d4bbfb70bbc355f29515a, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/f58f91a6c9c04086ab52723f8275480c] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp, totalSize=139.0 K 2024-11-21T21:17:01,417 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:17:01,418 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. files: [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/b3d2f126ca9e426aa9f07e55b8169554, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/1ed8f3988ef54f8287e273b0d1d1e037, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/f932256da73d4bbfb70bbc355f29515a, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/f58f91a6c9c04086ab52723f8275480c] 2024-11-21T21:17:01,418 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:17:01,418 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): 5a7f333184706b95cbb8c49fb3d84f31/B is initiating minor compaction (all files) 2024-11-21T21:17:01,418 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a7f333184706b95cbb8c49fb3d84f31/B in TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:17:01,418 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/3bbb1c80e2bc422c8f58d1b4e8035c0e, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/b3205ffaa342496eb7c6911e368b537e, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/2a9f1c94433f4068b5b9911943a67636, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/9b3e628cb0e4438e9092a6b86ae64ba5] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp, totalSize=48.1 K 2024-11-21T21:17:01,418 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3d2f126ca9e426aa9f07e55b8169554, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732223816027 2024-11-21T21:17:01,418 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bbb1c80e2bc422c8f58d1b4e8035c0e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732223816027 2024-11-21T21:17:01,418 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ed8f3988ef54f8287e273b0d1d1e037, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732223817189 2024-11-21T21:17:01,418 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting b3205ffaa342496eb7c6911e368b537e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732223817195 2024-11-21T21:17:01,419 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting f932256da73d4bbfb70bbc355f29515a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732223817856 2024-11-21T21:17:01,419 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a9f1c94433f4068b5b9911943a67636, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732223817856 2024-11-21T21:17:01,419 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] compactions.Compactor(225): Compacting f58f91a6c9c04086ab52723f8275480c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732223818545 2024-11-21T21:17:01,419 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b3e628cb0e4438e9092a6b86ae64ba5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732223818545 2024-11-21T21:17:01,426 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5a7f333184706b95cbb8c49fb3d84f31] 2024-11-21T21:17:01,428 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a7f333184706b95cbb8c49fb3d84f31#B#compaction#539 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:17:01,429 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/dc0dd17f9d4b48668fa0a164e5b1e871 is 50, key is test_row_0/B:col10/1732223818545/Put/seqid=0 2024-11-21T21:17:01,430 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112110bfddedd1ba4c89bdd2debd65b26885_5a7f333184706b95cbb8c49fb3d84f31 store=[table=TestAcidGuarantees family=A region=5a7f333184706b95cbb8c49fb3d84f31] 2024-11-21T21:17:01,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742454_1630 (size=12949) 2024-11-21T21:17:01,432 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112110bfddedd1ba4c89bdd2debd65b26885_5a7f333184706b95cbb8c49fb3d84f31, store=[table=TestAcidGuarantees family=A region=5a7f333184706b95cbb8c49fb3d84f31] 2024-11-21T21:17:01,433 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112110bfddedd1ba4c89bdd2debd65b26885_5a7f333184706b95cbb8c49fb3d84f31 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5a7f333184706b95cbb8c49fb3d84f31] 2024-11-21T21:17:01,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742455_1631 (size=4469) 2024-11-21T21:17:01,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-21T21:17:01,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:17:01,551 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2902): Flushing 5a7f333184706b95cbb8c49fb3d84f31 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-21T21:17:01,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=A 2024-11-21T21:17:01,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:17:01,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=B 2024-11-21T21:17:01,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:17:01,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=C 2024-11-21T21:17:01,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:17:01,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112128c8d83ad1d54cb3a38296062e8a9197_5a7f333184706b95cbb8c49fb3d84f31 is 50, key is test_row_0/A:col10/1732223819746/Put/seqid=0 2024-11-21T21:17:01,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742456_1632 (size=12454) 2024-11-21T21:17:01,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:17:01,563 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112128c8d83ad1d54cb3a38296062e8a9197_5a7f333184706b95cbb8c49fb3d84f31 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112128c8d83ad1d54cb3a38296062e8a9197_5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:17:01,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/b959d1827b52412ea69e69363d93a978, store: [table=TestAcidGuarantees family=A region=5a7f333184706b95cbb8c49fb3d84f31] 2024-11-21T21:17:01,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/b959d1827b52412ea69e69363d93a978 is 175, key is test_row_0/A:col10/1732223819746/Put/seqid=0 2024-11-21T21:17:01,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742457_1633 (size=31255) 2024-11-21T21:17:01,806 INFO [Thread-2473 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2473. 2024-11-21T21:17:01,806 DEBUG [Thread-2473 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:17:01,806 DEBUG [Thread-2473 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:17:01,806 DEBUG [Thread-2473 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:17:01,807 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:17:01,807 INFO [Thread-2471 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2471. 2024-11-21T21:17:01,807 DEBUG [Thread-2471 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:17:01,807 DEBUG [Thread-2471 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:17:01,807 DEBUG [Thread-2471 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:17:01,807 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:17:01,808 INFO [Thread-2467 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2467. 2024-11-21T21:17:01,808 DEBUG [Thread-2467 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:17:01,808 DEBUG [Thread-2467 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:17:01,808 DEBUG [Thread-2467 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:17:01,808 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:17:01,810 INFO [Thread-2469 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2469. 2024-11-21T21:17:01,810 DEBUG [Thread-2469 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:17:01,810 DEBUG [Thread-2469 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:17:01,810 DEBUG [Thread-2469 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:17:01,810 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:17:01,810 INFO [Thread-2475 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2475. 2024-11-21T21:17:01,810 DEBUG [Thread-2475 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:17:01,810 DEBUG [Thread-2475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:17:01,810 DEBUG [Thread-2475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:17:01,810 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:17:01,836 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a7f333184706b95cbb8c49fb3d84f31#A#compaction#540 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:17:01,836 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/dc0dd17f9d4b48668fa0a164e5b1e871 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/dc0dd17f9d4b48668fa0a164e5b1e871 2024-11-21T21:17:01,836 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/da78926f894c4f488225606eda90d1fd is 175, key is test_row_0/A:col10/1732223818545/Put/seqid=0 2024-11-21T21:17:01,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742458_1634 (size=31903) 2024-11-21T21:17:01,840 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5a7f333184706b95cbb8c49fb3d84f31/B of 5a7f333184706b95cbb8c49fb3d84f31 into dc0dd17f9d4b48668fa0a164e5b1e871(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:17:01,840 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a7f333184706b95cbb8c49fb3d84f31: 2024-11-21T21:17:01,841 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., storeName=5a7f333184706b95cbb8c49fb3d84f31/B, priority=12, startTime=1732223821416; duration=0sec 2024-11-21T21:17:01,841 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T21:17:01,841 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a7f333184706b95cbb8c49fb3d84f31:B 2024-11-21T21:17:01,841 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T21:17:01,842 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T21:17:01,842 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1541): 5a7f333184706b95cbb8c49fb3d84f31/C is initiating minor compaction (all files) 2024-11-21T21:17:01,842 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5a7f333184706b95cbb8c49fb3d84f31/C in TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:17:01,842 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/1703c1312c72449390654bf2b39c8a18, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/f916b2825cb342e6966d4f10e84177fa, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/ce33ebb2c40b4ec3a691e17a69cbf713, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/ce38058bcb404ab38b22f9d45e2ef8ee] into tmpdir=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp, totalSize=48.1 K 2024-11-21T21:17:01,842 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting 1703c1312c72449390654bf2b39c8a18, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732223816027 2024-11-21T21:17:01,842 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting f916b2825cb342e6966d4f10e84177fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732223817195 2024-11-21T21:17:01,842 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting ce33ebb2c40b4ec3a691e17a69cbf713, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732223817856 2024-11-21T21:17:01,843 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] compactions.Compactor(225): Compacting ce38058bcb404ab38b22f9d45e2ef8ee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732223818545 2024-11-21T21:17:01,849 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5a7f333184706b95cbb8c49fb3d84f31#C#compaction#542 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T21:17:01,849 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/2712a8a5ccae486dbcb53db4b9f84ab5 is 50, key is test_row_0/C:col10/1732223818545/Put/seqid=0 2024-11-21T21:17:01,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742459_1635 (size=12949) 2024-11-21T21:17:01,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41213 {}] regionserver.HRegion(8855): Flush requested on 5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:17:01,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. as already flushing 2024-11-21T21:17:01,902 INFO [Thread-2462 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2462. 2024-11-21T21:17:01,902 INFO [Thread-2460 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2460. 2024-11-21T21:17:01,902 INFO [Thread-2458 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2458. 2024-11-21T21:17:01,902 DEBUG [Thread-2460 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:17:01,902 DEBUG [Thread-2458 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:17:01,902 DEBUG [Thread-2462 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:17:01,902 DEBUG [Thread-2460 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:17:01,902 DEBUG [Thread-2458 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:17:01,902 DEBUG [Thread-2462 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:17:01,902 DEBUG [Thread-2460 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:17:01,902 DEBUG [Thread-2458 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:17:01,902 DEBUG [Thread-2462 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:17:01,902 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:17:01,902 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:17:01,902 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:17:01,912 INFO [Thread-2464 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2464. 2024-11-21T21:17:01,912 DEBUG [Thread-2464 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:17:01,912 DEBUG [Thread-2464 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:17:01,912 DEBUG [Thread-2464 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:17:01,912 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:17:01,968 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=289, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/b959d1827b52412ea69e69363d93a978 2024-11-21T21:17:01,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/1b2a8dd4b12b4774bed1bc3bbb1e012f is 50, key is test_row_0/B:col10/1732223819746/Put/seqid=0 2024-11-21T21:17:01,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742460_1636 (size=12301) 2024-11-21T21:17:02,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-21T21:17:02,252 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/da78926f894c4f488225606eda90d1fd as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/da78926f894c4f488225606eda90d1fd 2024-11-21T21:17:02,258 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5a7f333184706b95cbb8c49fb3d84f31/A of 5a7f333184706b95cbb8c49fb3d84f31 into da78926f894c4f488225606eda90d1fd(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:17:02,258 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a7f333184706b95cbb8c49fb3d84f31: 2024-11-21T21:17:02,258 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/2712a8a5ccae486dbcb53db4b9f84ab5 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/2712a8a5ccae486dbcb53db4b9f84ab5 2024-11-21T21:17:02,258 INFO [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., storeName=5a7f333184706b95cbb8c49fb3d84f31/A, priority=12, startTime=1732223821416; duration=0sec 2024-11-21T21:17:02,258 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:17:02,258 DEBUG [RS:0;849dee24a242:41213-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a7f333184706b95cbb8c49fb3d84f31:A 2024-11-21T21:17:02,263 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5a7f333184706b95cbb8c49fb3d84f31/C of 5a7f333184706b95cbb8c49fb3d84f31 into 2712a8a5ccae486dbcb53db4b9f84ab5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T21:17:02,263 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5a7f333184706b95cbb8c49fb3d84f31: 2024-11-21T21:17:02,263 INFO [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., storeName=5a7f333184706b95cbb8c49fb3d84f31/C, priority=12, startTime=1732223821416; duration=0sec 2024-11-21T21:17:02,263 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T21:17:02,263 DEBUG [RS:0;849dee24a242:41213-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5a7f333184706b95cbb8c49fb3d84f31:C 2024-11-21T21:17:02,377 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/1b2a8dd4b12b4774bed1bc3bbb1e012f 2024-11-21T21:17:02,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/19c19fa6ee594e7fa6a7a20a93271677 is 50, key is test_row_0/C:col10/1732223819746/Put/seqid=0 2024-11-21T21:17:02,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742461_1637 (size=12301) 2024-11-21T21:17:02,788 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/19c19fa6ee594e7fa6a7a20a93271677 2024-11-21T21:17:02,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/b959d1827b52412ea69e69363d93a978 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/b959d1827b52412ea69e69363d93a978 2024-11-21T21:17:02,802 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/b959d1827b52412ea69e69363d93a978, entries=150, sequenceid=289, filesize=30.5 K 2024-11-21T21:17:02,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/1b2a8dd4b12b4774bed1bc3bbb1e012f as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/1b2a8dd4b12b4774bed1bc3bbb1e012f 2024-11-21T21:17:02,806 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/1b2a8dd4b12b4774bed1bc3bbb1e012f, entries=150, sequenceid=289, filesize=12.0 K 2024-11-21T21:17:02,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/19c19fa6ee594e7fa6a7a20a93271677 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/19c19fa6ee594e7fa6a7a20a93271677 2024-11-21T21:17:02,810 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/19c19fa6ee594e7fa6a7a20a93271677, entries=150, sequenceid=289, filesize=12.0 K 2024-11-21T21:17:02,811 INFO [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=26.84 KB/27480 for 5a7f333184706b95cbb8c49fb3d84f31 in 1259ms, sequenceid=289, compaction requested=false 2024-11-21T21:17:02,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2603): Flush status journal for 5a7f333184706b95cbb8c49fb3d84f31: 2024-11-21T21:17:02,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:17:02,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/849dee24a242:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=175 2024-11-21T21:17:02,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster(4169): Remote procedure done, pid=175 2024-11-21T21:17:02,813 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=175, resume processing ppid=174 2024-11-21T21:17:02,813 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=175, ppid=174, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7890 sec 2024-11-21T21:17:02,815 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees in 2.7930 sec 2024-11-21T21:17:03,105 INFO [Thread-2456 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2456. 2024-11-21T21:17:03,105 DEBUG [Thread-2456 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T21:17:03,106 DEBUG [Thread-2456 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:17:03,106 DEBUG [Thread-2456 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:17:03,106 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:17:04,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-21T21:17:04,190 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:17:04,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T21:17:04,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 37 2024-11-21T21:17:04,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-21T21:17:04,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-21T21:17:04,191 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-21T21:17:04,191 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-21T21:17:04,191 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T21:17:04,191 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2997 2024-11-21T21:17:04,191 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2944 2024-11-21T21:17:04,191 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2991 2024-11-21T21:17:04,191 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3000 2024-11-21T21:17:04,191 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2950 2024-11-21T21:17:04,191 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T21:17:04,191 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T21:17:04,191 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T21:17:04,191 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:17:04,191 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:17:04,191 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:17:04,192 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-21T21:17:04,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=176, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T21:17:04,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-21T21:17:04,194 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732223824194"}]},"ts":"1732223824194"} 2024-11-21T21:17:04,195 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T21:17:04,196 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T21:17:04,196 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=177, ppid=176, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T21:17:04,197 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=178, ppid=177, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5a7f333184706b95cbb8c49fb3d84f31, UNASSIGN}] 2024-11-21T21:17:04,198 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=178, ppid=177, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5a7f333184706b95cbb8c49fb3d84f31, UNASSIGN 2024-11-21T21:17:04,199 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=178 updating hbase:meta row=5a7f333184706b95cbb8c49fb3d84f31, regionState=CLOSING, regionLocation=849dee24a242,41213,1732223653469 2024-11-21T21:17:04,200 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=178, ppid=177, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5a7f333184706b95cbb8c49fb3d84f31, UNASSIGN because future has completed 2024-11-21T21:17:04,200 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T21:17:04,200 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=179, ppid=178, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469}] 2024-11-21T21:17:04,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-21T21:17:04,353 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] handler.UnassignRegionHandler(122): Close 5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:17:04,353 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T21:17:04,353 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(1722): Closing 5a7f333184706b95cbb8c49fb3d84f31, disabling compactions & flushes 2024-11-21T21:17:04,353 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:17:04,353 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:17:04,353 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. after waiting 0 ms 2024-11-21T21:17:04,353 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:17:04,353 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(2902): Flushing 5a7f333184706b95cbb8c49fb3d84f31 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-21T21:17:04,353 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=A 2024-11-21T21:17:04,353 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:17:04,353 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=B 2024-11-21T21:17:04,353 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:17:04,353 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 5a7f333184706b95cbb8c49fb3d84f31, store=C 2024-11-21T21:17:04,354 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-21T21:17:04,362 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d01a946c1b064736878e6777fd306033_5a7f333184706b95cbb8c49fb3d84f31 is 50, key is test_row_0/A:col10/1732223821900/Put/seqid=0 2024-11-21T21:17:04,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742462_1638 (size=12454) 2024-11-21T21:17:04,367 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T21:17:04,370 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d01a946c1b064736878e6777fd306033_5a7f333184706b95cbb8c49fb3d84f31 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d01a946c1b064736878e6777fd306033_5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:17:04,371 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/795750c0a99d475992406250bcaa4ea6, store: [table=TestAcidGuarantees family=A region=5a7f333184706b95cbb8c49fb3d84f31] 2024-11-21T21:17:04,372 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/795750c0a99d475992406250bcaa4ea6 is 175, key is test_row_0/A:col10/1732223821900/Put/seqid=0 2024-11-21T21:17:04,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742463_1639 (size=31255) 2024-11-21T21:17:04,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-21T21:17:04,777 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=300, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/795750c0a99d475992406250bcaa4ea6 2024-11-21T21:17:04,789 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/a10af56a597b4295af09aa82dbcf73c1 is 50, key is test_row_0/B:col10/1732223821900/Put/seqid=0 2024-11-21T21:17:04,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742464_1640 (size=12301) 2024-11-21T21:17:04,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-21T21:17:05,194 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/a10af56a597b4295af09aa82dbcf73c1 2024-11-21T21:17:05,203 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/d6b4cdd3caf646e2adb5bad2c7d8b1cd is 50, key is test_row_0/C:col10/1732223821900/Put/seqid=0 2024-11-21T21:17:05,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742465_1641 (size=12301) 2024-11-21T21:17:05,209 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/d6b4cdd3caf646e2adb5bad2c7d8b1cd 2024-11-21T21:17:05,213 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/A/795750c0a99d475992406250bcaa4ea6 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/795750c0a99d475992406250bcaa4ea6 2024-11-21T21:17:05,216 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/795750c0a99d475992406250bcaa4ea6, entries=150, sequenceid=300, filesize=30.5 K 2024-11-21T21:17:05,216 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/B/a10af56a597b4295af09aa82dbcf73c1 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/a10af56a597b4295af09aa82dbcf73c1 2024-11-21T21:17:05,218 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/a10af56a597b4295af09aa82dbcf73c1, entries=150, sequenceid=300, filesize=12.0 K 2024-11-21T21:17:05,219 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/.tmp/C/d6b4cdd3caf646e2adb5bad2c7d8b1cd as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/d6b4cdd3caf646e2adb5bad2c7d8b1cd 2024-11-21T21:17:05,221 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/d6b4cdd3caf646e2adb5bad2c7d8b1cd, entries=150, sequenceid=300, filesize=12.0 K 2024-11-21T21:17:05,221 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 5a7f333184706b95cbb8c49fb3d84f31 in 868ms, sequenceid=300, compaction requested=true 2024-11-21T21:17:05,222 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/4343f0b9f95e46c2a1c3b2b14a698fa3, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/bf6bbbb7668248e3a5a9ca45f58891ad, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/2f5e69d5955e4903a30cf68ea0aafe9d, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/f785e28c91604991a62412b2f87f110f, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/948454736c674488a24016b9fc20517a, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/5d9d2ed2bf3142199f6b50af11c3bf32, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/1650a21e8ae74536ae9e00c65baf77f6, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/2f56ccbb652c4c7baca25ebc2c078cca, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/dce2c58ff0d54c018929ad88963b0015, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/5ca66206a0a24ce0b06e119430f6456b, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/a9fa2e0497854388afd89b74ba9699fb, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/9c5914a709b74718b344ed628a362fca, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/166e6615572e4ef9bcf107ff526937f5, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/2fe9ef390d2848b0b071ee97df22eedd, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/d161e76da3334125bcf42dff9dff516b, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/b3d2f126ca9e426aa9f07e55b8169554, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/1ed8f3988ef54f8287e273b0d1d1e037, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/f932256da73d4bbfb70bbc355f29515a, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/f58f91a6c9c04086ab52723f8275480c] to archive 2024-11-21T21:17:05,222 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T21:17:05,223 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/4343f0b9f95e46c2a1c3b2b14a698fa3 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/4343f0b9f95e46c2a1c3b2b14a698fa3 2024-11-21T21:17:05,224 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/bf6bbbb7668248e3a5a9ca45f58891ad to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/bf6bbbb7668248e3a5a9ca45f58891ad 2024-11-21T21:17:05,225 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/2f5e69d5955e4903a30cf68ea0aafe9d to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/2f5e69d5955e4903a30cf68ea0aafe9d 2024-11-21T21:17:05,225 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/f785e28c91604991a62412b2f87f110f to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/f785e28c91604991a62412b2f87f110f 2024-11-21T21:17:05,226 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/948454736c674488a24016b9fc20517a to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/948454736c674488a24016b9fc20517a 2024-11-21T21:17:05,227 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/5d9d2ed2bf3142199f6b50af11c3bf32 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/5d9d2ed2bf3142199f6b50af11c3bf32 2024-11-21T21:17:05,227 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/1650a21e8ae74536ae9e00c65baf77f6 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/1650a21e8ae74536ae9e00c65baf77f6 2024-11-21T21:17:05,228 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/2f56ccbb652c4c7baca25ebc2c078cca to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/2f56ccbb652c4c7baca25ebc2c078cca 2024-11-21T21:17:05,229 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/dce2c58ff0d54c018929ad88963b0015 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/dce2c58ff0d54c018929ad88963b0015 2024-11-21T21:17:05,230 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/5ca66206a0a24ce0b06e119430f6456b to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/5ca66206a0a24ce0b06e119430f6456b 2024-11-21T21:17:05,231 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/a9fa2e0497854388afd89b74ba9699fb to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/a9fa2e0497854388afd89b74ba9699fb 2024-11-21T21:17:05,232 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/9c5914a709b74718b344ed628a362fca to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/9c5914a709b74718b344ed628a362fca 2024-11-21T21:17:05,232 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/166e6615572e4ef9bcf107ff526937f5 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/166e6615572e4ef9bcf107ff526937f5 2024-11-21T21:17:05,233 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/2fe9ef390d2848b0b071ee97df22eedd to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/2fe9ef390d2848b0b071ee97df22eedd 2024-11-21T21:17:05,234 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/d161e76da3334125bcf42dff9dff516b to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/d161e76da3334125bcf42dff9dff516b 2024-11-21T21:17:05,235 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/b3d2f126ca9e426aa9f07e55b8169554 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/b3d2f126ca9e426aa9f07e55b8169554 2024-11-21T21:17:05,236 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/1ed8f3988ef54f8287e273b0d1d1e037 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/1ed8f3988ef54f8287e273b0d1d1e037 2024-11-21T21:17:05,237 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/f932256da73d4bbfb70bbc355f29515a to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/f932256da73d4bbfb70bbc355f29515a 2024-11-21T21:17:05,238 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/f58f91a6c9c04086ab52723f8275480c to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/f58f91a6c9c04086ab52723f8275480c 2024-11-21T21:17:05,239 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/e4e2205110ec4855a7f9177b4a5a4769, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/79bd7201ea7c4469a6154524bdff8ca8, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/9763ad8ab1e34eb9b6f76129cb3c26f1, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/1c0e70eb910e4402b0d897461d8f539e, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/a6a4399ad3a845449361005bacedccb2, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/ea143fd9542a4ecfb40498b059b03b04, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/ec8dde1ab3a943a6b6d6821abbc0ed54, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/008e499b9d7b438cbc36bfd346f25c03, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/b64e473b88854a3b8f8e333eb54a338e, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/be7ea0a0a7a24d17a29a948bca3ed17e, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/054d2e7c581048dba4ac2fd4e35b5f23, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/3f5183e7b33245e98af228f682fcfc2c, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/ca71f34441cc4a2fa6f29c50cd4b2a97, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/a560ff0448fe4de2be7d16a993c6ef99, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/3bbb1c80e2bc422c8f58d1b4e8035c0e, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/c4a80005f623433cba6110dd4e86b35e, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/b3205ffaa342496eb7c6911e368b537e, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/2a9f1c94433f4068b5b9911943a67636, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/9b3e628cb0e4438e9092a6b86ae64ba5] to archive 2024-11-21T21:17:05,240 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T21:17:05,242 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/e4e2205110ec4855a7f9177b4a5a4769 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/e4e2205110ec4855a7f9177b4a5a4769 2024-11-21T21:17:05,243 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/79bd7201ea7c4469a6154524bdff8ca8 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/79bd7201ea7c4469a6154524bdff8ca8 2024-11-21T21:17:05,244 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/9763ad8ab1e34eb9b6f76129cb3c26f1 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/9763ad8ab1e34eb9b6f76129cb3c26f1 2024-11-21T21:17:05,245 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/1c0e70eb910e4402b0d897461d8f539e to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/1c0e70eb910e4402b0d897461d8f539e 2024-11-21T21:17:05,247 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/a6a4399ad3a845449361005bacedccb2 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/a6a4399ad3a845449361005bacedccb2 2024-11-21T21:17:05,248 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/ea143fd9542a4ecfb40498b059b03b04 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/ea143fd9542a4ecfb40498b059b03b04 2024-11-21T21:17:05,249 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/ec8dde1ab3a943a6b6d6821abbc0ed54 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/ec8dde1ab3a943a6b6d6821abbc0ed54 2024-11-21T21:17:05,251 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/008e499b9d7b438cbc36bfd346f25c03 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/008e499b9d7b438cbc36bfd346f25c03 2024-11-21T21:17:05,252 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/b64e473b88854a3b8f8e333eb54a338e to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/b64e473b88854a3b8f8e333eb54a338e 2024-11-21T21:17:05,253 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/be7ea0a0a7a24d17a29a948bca3ed17e to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/be7ea0a0a7a24d17a29a948bca3ed17e 2024-11-21T21:17:05,254 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/054d2e7c581048dba4ac2fd4e35b5f23 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/054d2e7c581048dba4ac2fd4e35b5f23 2024-11-21T21:17:05,255 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/3f5183e7b33245e98af228f682fcfc2c to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/3f5183e7b33245e98af228f682fcfc2c 2024-11-21T21:17:05,257 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/ca71f34441cc4a2fa6f29c50cd4b2a97 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/ca71f34441cc4a2fa6f29c50cd4b2a97 2024-11-21T21:17:05,258 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/a560ff0448fe4de2be7d16a993c6ef99 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/a560ff0448fe4de2be7d16a993c6ef99 2024-11-21T21:17:05,259 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/3bbb1c80e2bc422c8f58d1b4e8035c0e to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/3bbb1c80e2bc422c8f58d1b4e8035c0e 2024-11-21T21:17:05,260 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/c4a80005f623433cba6110dd4e86b35e to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/c4a80005f623433cba6110dd4e86b35e 2024-11-21T21:17:05,261 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/b3205ffaa342496eb7c6911e368b537e to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/b3205ffaa342496eb7c6911e368b537e 2024-11-21T21:17:05,262 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/2a9f1c94433f4068b5b9911943a67636 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/2a9f1c94433f4068b5b9911943a67636 2024-11-21T21:17:05,262 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/9b3e628cb0e4438e9092a6b86ae64ba5 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/9b3e628cb0e4438e9092a6b86ae64ba5 2024-11-21T21:17:05,263 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/03616e13e463456783f8dd38f7bb46a4, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/02bb581151d64d4fa2559aaf82c295c0, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/c4c0042840664f368aa2766ef2c6cd33, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/19f425bf2ed74bf89aa4ddba9acfa1f8, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/17b7277b32c345b0a9563814c5a0e621, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/455b036b27c84c8cb7656886d1b7c966, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/4758f5d824744ae2aeddbc496eccafbc, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/fe1b07463ef34941b7e591bb193c8551, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/cbbe4ce722ca4b8b9e0adab7173e647c, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/b71fd2469dd84bdfb8de9667972f61e7, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/cdd3a7530e204b0ea296aa675b6294d8, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/60c1c0de92d449b0b8e47b1e2ce6698f, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/f28150ce24144b4e9eafb74783169fd1, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/842827bcbe9a4cb3b3294c5ff83c72c1, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/1703c1312c72449390654bf2b39c8a18, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/33b44a57b06c4b68b8ecece66fc375c1, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/f916b2825cb342e6966d4f10e84177fa, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/ce33ebb2c40b4ec3a691e17a69cbf713, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/ce38058bcb404ab38b22f9d45e2ef8ee] to archive 2024-11-21T21:17:05,264 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T21:17:05,265 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/03616e13e463456783f8dd38f7bb46a4 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/03616e13e463456783f8dd38f7bb46a4 2024-11-21T21:17:05,265 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/02bb581151d64d4fa2559aaf82c295c0 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/02bb581151d64d4fa2559aaf82c295c0 2024-11-21T21:17:05,266 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/c4c0042840664f368aa2766ef2c6cd33 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/c4c0042840664f368aa2766ef2c6cd33 2024-11-21T21:17:05,267 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/19f425bf2ed74bf89aa4ddba9acfa1f8 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/19f425bf2ed74bf89aa4ddba9acfa1f8 2024-11-21T21:17:05,267 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/17b7277b32c345b0a9563814c5a0e621 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/17b7277b32c345b0a9563814c5a0e621 2024-11-21T21:17:05,268 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/455b036b27c84c8cb7656886d1b7c966 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/455b036b27c84c8cb7656886d1b7c966 2024-11-21T21:17:05,268 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/4758f5d824744ae2aeddbc496eccafbc to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/4758f5d824744ae2aeddbc496eccafbc 2024-11-21T21:17:05,269 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/fe1b07463ef34941b7e591bb193c8551 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/fe1b07463ef34941b7e591bb193c8551 2024-11-21T21:17:05,270 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/cbbe4ce722ca4b8b9e0adab7173e647c to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/cbbe4ce722ca4b8b9e0adab7173e647c 2024-11-21T21:17:05,270 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/b71fd2469dd84bdfb8de9667972f61e7 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/b71fd2469dd84bdfb8de9667972f61e7 2024-11-21T21:17:05,271 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/cdd3a7530e204b0ea296aa675b6294d8 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/cdd3a7530e204b0ea296aa675b6294d8 2024-11-21T21:17:05,271 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/60c1c0de92d449b0b8e47b1e2ce6698f to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/60c1c0de92d449b0b8e47b1e2ce6698f 2024-11-21T21:17:05,272 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/f28150ce24144b4e9eafb74783169fd1 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/f28150ce24144b4e9eafb74783169fd1 2024-11-21T21:17:05,273 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/842827bcbe9a4cb3b3294c5ff83c72c1 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/842827bcbe9a4cb3b3294c5ff83c72c1 2024-11-21T21:17:05,274 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/1703c1312c72449390654bf2b39c8a18 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/1703c1312c72449390654bf2b39c8a18 2024-11-21T21:17:05,275 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/33b44a57b06c4b68b8ecece66fc375c1 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/33b44a57b06c4b68b8ecece66fc375c1 2024-11-21T21:17:05,276 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/f916b2825cb342e6966d4f10e84177fa to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/f916b2825cb342e6966d4f10e84177fa 2024-11-21T21:17:05,276 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/ce33ebb2c40b4ec3a691e17a69cbf713 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/ce33ebb2c40b4ec3a691e17a69cbf713 2024-11-21T21:17:05,277 DEBUG [StoreCloser-TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/ce38058bcb404ab38b22f9d45e2ef8ee to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/ce38058bcb404ab38b22f9d45e2ef8ee 2024-11-21T21:17:05,281 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/recovered.edits/303.seqid, newMaxSeqId=303, maxSeqId=4 2024-11-21T21:17:05,281 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. 2024-11-21T21:17:05,281 DEBUG [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(1676): Region close journal for 5a7f333184706b95cbb8c49fb3d84f31: Waiting for close lock at 1732223824353Running coprocessor pre-close hooks at 1732223824353Disabling compacts and flushes for region at 1732223824353Disabling writes for close at 1732223824353Obtaining lock to block concurrent updates at 1732223824353Preparing flush snapshotting stores in 5a7f333184706b95cbb8c49fb3d84f31 at 1732223824353Finished memstore snapshotting TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732223824354 (+1 ms)Flushing stores of TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31. at 1732223824354Flushing 5a7f333184706b95cbb8c49fb3d84f31/A: creating writer at 1732223824354Flushing mob file 5a7f333184706b95cbb8c49fb3d84f31/A: appending metadata at 1732223824361 (+7 ms)Flushing mob file 5a7f333184706b95cbb8c49fb3d84f31/A: closing flushed file at 1732223824361Flushing 5a7f333184706b95cbb8c49fb3d84f31/A: appending metadata at 1732223824371 (+10 ms)Flushing 5a7f333184706b95cbb8c49fb3d84f31/A: closing flushed file at 1732223824371Flushing 5a7f333184706b95cbb8c49fb3d84f31/B: creating writer at 1732223824786 (+415 ms)Flushing 5a7f333184706b95cbb8c49fb3d84f31/B: appending metadata at 1732223824788 (+2 ms)Flushing 5a7f333184706b95cbb8c49fb3d84f31/B: closing flushed file at 1732223824789 (+1 ms)Flushing 5a7f333184706b95cbb8c49fb3d84f31/C: creating writer at 1732223825200 (+411 ms)Flushing 5a7f333184706b95cbb8c49fb3d84f31/C: appending metadata at 1732223825203 (+3 ms)Flushing 5a7f333184706b95cbb8c49fb3d84f31/C: closing flushed file at 1732223825203Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@185a86ff: reopening flushed file at 1732223825212 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d1db474: reopening flushed file at 1732223825216 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3bbfa819: reopening flushed file at 1732223825218 (+2 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 5a7f333184706b95cbb8c49fb3d84f31 in 868ms, sequenceid=300, compaction requested=true at 1732223825221 (+3 ms)Writing region close event to WAL at 1732223825278 (+57 ms)Running coprocessor post-close hooks at 1732223825281 (+3 ms)Closed at 1732223825281 2024-11-21T21:17:05,283 INFO [RS_CLOSE_REGION-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] handler.UnassignRegionHandler(157): Closed 5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:17:05,283 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=178 updating hbase:meta row=5a7f333184706b95cbb8c49fb3d84f31, regionState=CLOSED 2024-11-21T21:17:05,285 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=179, ppid=178, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 because future has completed 2024-11-21T21:17:05,287 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=179, resume processing ppid=178 2024-11-21T21:17:05,287 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=179, ppid=178, state=SUCCESS, hasLock=false; CloseRegionProcedure 5a7f333184706b95cbb8c49fb3d84f31, server=849dee24a242,41213,1732223653469 in 1.0850 sec 2024-11-21T21:17:05,289 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=178, resume processing ppid=177 2024-11-21T21:17:05,289 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=178, ppid=177, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5a7f333184706b95cbb8c49fb3d84f31, UNASSIGN in 1.0900 sec 2024-11-21T21:17:05,291 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=177, resume processing ppid=176 2024-11-21T21:17:05,291 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=177, ppid=176, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.0940 sec 2024-11-21T21:17:05,293 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732223825292"}]},"ts":"1732223825292"} 2024-11-21T21:17:05,294 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T21:17:05,294 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T21:17:05,297 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=176, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.1030 sec 2024-11-21T21:17:05,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-21T21:17:05,331 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:17:05,332 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-21T21:17:05,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] procedure2.ProcedureExecutor(1139): Stored pid=180, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:17:05,334 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=180, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:17:05,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-21T21:17:05,335 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=180, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:17:05,339 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:17:05,343 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A, FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B, FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C, FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/recovered.edits] 2024-11-21T21:17:05,347 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/795750c0a99d475992406250bcaa4ea6 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/795750c0a99d475992406250bcaa4ea6 2024-11-21T21:17:05,349 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/b959d1827b52412ea69e69363d93a978 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/b959d1827b52412ea69e69363d93a978 2024-11-21T21:17:05,351 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/da78926f894c4f488225606eda90d1fd to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/A/da78926f894c4f488225606eda90d1fd 2024-11-21T21:17:05,355 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/1b2a8dd4b12b4774bed1bc3bbb1e012f to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/1b2a8dd4b12b4774bed1bc3bbb1e012f 2024-11-21T21:17:05,357 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/a10af56a597b4295af09aa82dbcf73c1 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/a10af56a597b4295af09aa82dbcf73c1 2024-11-21T21:17:05,360 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/dc0dd17f9d4b48668fa0a164e5b1e871 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/B/dc0dd17f9d4b48668fa0a164e5b1e871 2024-11-21T21:17:05,365 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/19c19fa6ee594e7fa6a7a20a93271677 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/19c19fa6ee594e7fa6a7a20a93271677 2024-11-21T21:17:05,366 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/2712a8a5ccae486dbcb53db4b9f84ab5 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/2712a8a5ccae486dbcb53db4b9f84ab5 2024-11-21T21:17:05,368 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/d6b4cdd3caf646e2adb5bad2c7d8b1cd to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/C/d6b4cdd3caf646e2adb5bad2c7d8b1cd 2024-11-21T21:17:05,371 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/recovered.edits/303.seqid to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31/recovered.edits/303.seqid 2024-11-21T21:17:05,371 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/default/TestAcidGuarantees/5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:17:05,371 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T21:17:05,372 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T21:17:05,373 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-21T21:17:05,376 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411211ff8d56e501b43a4bdda6dd04a32a79f_5a7f333184706b95cbb8c49fb3d84f31 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411211ff8d56e501b43a4bdda6dd04a32a79f_5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:17:05,377 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212227f9983b2846238709c579e7a52a64_5a7f333184706b95cbb8c49fb3d84f31 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212227f9983b2846238709c579e7a52a64_5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:17:05,378 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212236ae25ee394813a23dd6013e712929_5a7f333184706b95cbb8c49fb3d84f31 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212236ae25ee394813a23dd6013e712929_5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:17:05,379 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112128c8d83ad1d54cb3a38296062e8a9197_5a7f333184706b95cbb8c49fb3d84f31 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112128c8d83ad1d54cb3a38296062e8a9197_5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:17:05,380 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214c325f35a3fe47df9172a8701f408d40_5a7f333184706b95cbb8c49fb3d84f31 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214c325f35a3fe47df9172a8701f408d40_5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:17:05,381 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112150468b23a2c6415bb2301197ec60e4c0_5a7f333184706b95cbb8c49fb3d84f31 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112150468b23a2c6415bb2301197ec60e4c0_5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:17:05,381 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112152d534af9e7f4081acb1a93169fe6a2c_5a7f333184706b95cbb8c49fb3d84f31 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112152d534af9e7f4081acb1a93169fe6a2c_5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:17:05,382 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112153067786bb594d029a4b5a9daba7e27b_5a7f333184706b95cbb8c49fb3d84f31 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112153067786bb594d029a4b5a9daba7e27b_5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:17:05,383 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216323518ff31a475191ff308d40cb0e62_5a7f333184706b95cbb8c49fb3d84f31 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216323518ff31a475191ff308d40cb0e62_5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:17:05,383 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216bc2ea10132c42d29bc32f20e3fb7fa5_5a7f333184706b95cbb8c49fb3d84f31 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216bc2ea10132c42d29bc32f20e3fb7fa5_5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:17:05,384 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a247247819e442e3b2719fe140e33b91_5a7f333184706b95cbb8c49fb3d84f31 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a247247819e442e3b2719fe140e33b91_5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:17:05,384 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a6bb73b7cb3a4af4aea030c25520a20b_5a7f333184706b95cbb8c49fb3d84f31 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a6bb73b7cb3a4af4aea030c25520a20b_5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:17:05,385 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b54e652040c94d21b34bbebdcb4b5c47_5a7f333184706b95cbb8c49fb3d84f31 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b54e652040c94d21b34bbebdcb4b5c47_5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:17:05,386 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ba5efa6366a4472f8f1f1180aa848c70_5a7f333184706b95cbb8c49fb3d84f31 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ba5efa6366a4472f8f1f1180aa848c70_5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:17:05,386 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d01a946c1b064736878e6777fd306033_5a7f333184706b95cbb8c49fb3d84f31 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d01a946c1b064736878e6777fd306033_5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:17:05,387 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121f0ef9a92aaa5427ab7cd5bd2c42717dc_5a7f333184706b95cbb8c49fb3d84f31 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121f0ef9a92aaa5427ab7cd5bd2c42717dc_5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:17:05,388 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121faae23ed521f46e5b1225875e5738fc5_5a7f333184706b95cbb8c49fb3d84f31 to hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121faae23ed521f46e5b1225875e5738fc5_5a7f333184706b95cbb8c49fb3d84f31 2024-11-21T21:17:05,388 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T21:17:05,390 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=180, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:17:05,392 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T21:17:05,394 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T21:17:05,395 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=180, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:17:05,395 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T21:17:05,396 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732223825395"}]},"ts":"9223372036854775807"} 2024-11-21T21:17:05,397 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T21:17:05,397 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 5a7f333184706b95cbb8c49fb3d84f31, NAME => 'TestAcidGuarantees,,1732223799657.5a7f333184706b95cbb8c49fb3d84f31.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T21:17:05,398 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T21:17:05,398 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732223825398"}]},"ts":"9223372036854775807"} 2024-11-21T21:17:05,399 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T21:17:05,400 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=180, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T21:17:05,401 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=180, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 68 msec 2024-11-21T21:17:05,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42485 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-21T21:17:05,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T21:17:05,441 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T21:17:05,449 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250 (was 250), OpenFileDescriptor=447 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=426 (was 403) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=12184 (was 11428) - AvailableMemoryMB LEAK? - 2024-11-21T21:17:05,449 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-21T21:17:05,449 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T21:17:05,449 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T21:17:05,449 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:17:05,449 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:17:05,449 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T21:17:05,449 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-21T21:17:05,449 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1945866401, stopped=false 2024-11-21T21:17:05,450 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=849dee24a242,42485,1732223652547 2024-11-21T21:17:05,484 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42485-0x1015f3f4af00000, quorum=127.0.0.1:52266, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T21:17:05,484 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41213-0x1015f3f4af00001, quorum=127.0.0.1:52266, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T21:17:05,484 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42485-0x1015f3f4af00000, quorum=127.0.0.1:52266, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:17:05,484 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41213-0x1015f3f4af00001, quorum=127.0.0.1:52266, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:17:05,484 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T21:17:05,485 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T21:17:05,485 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T21:17:05,485 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42485-0x1015f3f4af00000, quorum=127.0.0.1:52266, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T21:17:05,485 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:17:05,485 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41213-0x1015f3f4af00001, quorum=127.0.0.1:52266, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T21:17:05,485 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '849dee24a242,41213,1732223653469' ***** 2024-11-21T21:17:05,486 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-21T21:17:05,486 INFO [RS:0;849dee24a242:41213 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-21T21:17:05,486 INFO [RS:0;849dee24a242:41213 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-21T21:17:05,486 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-21T21:17:05,486 INFO [RS:0;849dee24a242:41213 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-21T21:17:05,486 INFO [RS:0;849dee24a242:41213 {}] regionserver.HRegionServer(959): stopping server 849dee24a242,41213,1732223653469 2024-11-21T21:17:05,486 INFO [RS:0;849dee24a242:41213 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T21:17:05,486 INFO [RS:0;849dee24a242:41213 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;849dee24a242:41213. 2024-11-21T21:17:05,487 DEBUG [RS:0;849dee24a242:41213 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T21:17:05,487 DEBUG [RS:0;849dee24a242:41213 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:17:05,487 INFO [RS:0;849dee24a242:41213 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-21T21:17:05,487 INFO [RS:0;849dee24a242:41213 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-21T21:17:05,487 INFO [RS:0;849dee24a242:41213 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-21T21:17:05,487 INFO [RS:0;849dee24a242:41213 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-21T21:17:05,487 INFO [RS:0;849dee24a242:41213 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-21T21:17:05,487 DEBUG [RS:0;849dee24a242:41213 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-21T21:17:05,488 DEBUG [RS:0;849dee24a242:41213 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T21:17:05,488 DEBUG [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T21:17:05,488 INFO [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T21:17:05,488 DEBUG [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T21:17:05,488 DEBUG [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T21:17:05,488 DEBUG [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T21:17:05,488 INFO [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-21T21:17:05,519 DEBUG [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/hbase/meta/1588230740/.tmp/info/c78af2e5bec34b86bddafab9a365c00a is 95, key is TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947./info:/1732223684981/DeleteFamily/seqid=0 2024-11-21T21:17:05,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742466_1642 (size=6514) 2024-11-21T21:17:05,543 INFO [regionserver/849dee24a242:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T21:17:05,688 DEBUG [RS:0;849dee24a242:41213 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T21:17:05,855 INFO [regionserver/849dee24a242:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-21T21:17:05,856 INFO [regionserver/849dee24a242:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-21T21:17:05,888 DEBUG [RS:0;849dee24a242:41213 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T21:17:05,922 INFO [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/hbase/meta/1588230740/.tmp/info/c78af2e5bec34b86bddafab9a365c00a 2024-11-21T21:17:05,942 DEBUG [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/hbase/meta/1588230740/.tmp/ns/a2d342530a7147188c723d5030bedda3 is 93, key is TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947./ns:/1732223684941/DeleteFamily/seqid=0 2024-11-21T21:17:05,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742467_1643 (size=5906) 2024-11-21T21:17:06,088 DEBUG [RS:0;849dee24a242:41213 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T21:17:06,289 DEBUG [RS:0;849dee24a242:41213 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T21:17:06,347 INFO [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/hbase/meta/1588230740/.tmp/ns/a2d342530a7147188c723d5030bedda3 2024-11-21T21:17:06,376 DEBUG [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/hbase/meta/1588230740/.tmp/rep_barrier/0aaa909f7b0f4bbbaf15240b11a19394 is 102, key is TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947./rep_barrier:/1732223684941/DeleteFamily/seqid=0 2024-11-21T21:17:06,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742468_1644 (size=6025) 2024-11-21T21:17:06,489 DEBUG [RS:0;849dee24a242:41213 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T21:17:06,689 DEBUG [RS:0;849dee24a242:41213 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T21:17:06,781 INFO [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/hbase/meta/1588230740/.tmp/rep_barrier/0aaa909f7b0f4bbbaf15240b11a19394 2024-11-21T21:17:06,806 DEBUG [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/hbase/meta/1588230740/.tmp/table/64a2a38fca9248f8ab08e08c10257d34 is 96, key is TestAcidGuarantees,,1732223658394.ae951cc3e18af3209220d547b1b63947./table:/1732223684941/DeleteFamily/seqid=0 2024-11-21T21:17:06,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742469_1645 (size=5936) 2024-11-21T21:17:06,809 INFO [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/hbase/meta/1588230740/.tmp/table/64a2a38fca9248f8ab08e08c10257d34 2024-11-21T21:17:06,812 DEBUG [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/hbase/meta/1588230740/.tmp/info/c78af2e5bec34b86bddafab9a365c00a as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/hbase/meta/1588230740/info/c78af2e5bec34b86bddafab9a365c00a 2024-11-21T21:17:06,815 INFO [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/hbase/meta/1588230740/info/c78af2e5bec34b86bddafab9a365c00a, entries=12, sequenceid=90, filesize=6.4 K 2024-11-21T21:17:06,816 DEBUG [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/hbase/meta/1588230740/.tmp/ns/a2d342530a7147188c723d5030bedda3 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/hbase/meta/1588230740/ns/a2d342530a7147188c723d5030bedda3 2024-11-21T21:17:06,819 INFO [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/hbase/meta/1588230740/ns/a2d342530a7147188c723d5030bedda3, entries=8, sequenceid=90, filesize=5.8 K 2024-11-21T21:17:06,820 DEBUG [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/hbase/meta/1588230740/.tmp/rep_barrier/0aaa909f7b0f4bbbaf15240b11a19394 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/hbase/meta/1588230740/rep_barrier/0aaa909f7b0f4bbbaf15240b11a19394 2024-11-21T21:17:06,822 INFO [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/hbase/meta/1588230740/rep_barrier/0aaa909f7b0f4bbbaf15240b11a19394, entries=6, sequenceid=90, filesize=5.9 K 2024-11-21T21:17:06,823 DEBUG [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/hbase/meta/1588230740/.tmp/table/64a2a38fca9248f8ab08e08c10257d34 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/hbase/meta/1588230740/table/64a2a38fca9248f8ab08e08c10257d34 2024-11-21T21:17:06,825 INFO [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/hbase/meta/1588230740/table/64a2a38fca9248f8ab08e08c10257d34, entries=7, sequenceid=90, filesize=5.8 K 2024-11-21T21:17:06,826 INFO [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1338ms, sequenceid=90, compaction requested=false 2024-11-21T21:17:06,829 DEBUG [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-21T21:17:06,830 DEBUG [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-21T21:17:06,830 INFO [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T21:17:06,830 DEBUG [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732223825487Running coprocessor pre-close hooks at 1732223825488 (+1 ms)Disabling compacts and flushes for region at 1732223825488Disabling writes for close at 1732223825488Obtaining lock to block concurrent updates at 1732223825488Preparing flush snapshotting stores in 1588230740 at 1732223825488Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732223825489 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732223825489Flushing 1588230740/info: creating writer at 1732223825490 (+1 ms)Flushing 1588230740/info: appending metadata at 1732223825518 (+28 ms)Flushing 1588230740/info: closing flushed file at 1732223825518Flushing 1588230740/ns: creating writer at 1732223825927 (+409 ms)Flushing 1588230740/ns: appending metadata at 1732223825942 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1732223825942Flushing 1588230740/rep_barrier: creating writer at 1732223826357 (+415 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732223826376 (+19 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732223826376Flushing 1588230740/table: creating writer at 1732223826789 (+413 ms)Flushing 1588230740/table: appending metadata at 1732223826805 (+16 ms)Flushing 1588230740/table: closing flushed file at 1732223826805Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d57638: reopening flushed file at 1732223826812 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@bee39a3: reopening flushed file at 1732223826816 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@470da393: reopening flushed file at 1732223826819 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f62488f: reopening flushed file at 1732223826822 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1338ms, sequenceid=90, compaction requested=false at 1732223826826 (+4 ms)Writing region close event to WAL at 1732223826827 (+1 ms)Running coprocessor post-close hooks at 1732223826830 (+3 ms)Closed at 1732223826830 2024-11-21T21:17:06,830 DEBUG [RS_CLOSE_META-regionserver/849dee24a242:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-21T21:17:06,889 INFO [RS:0;849dee24a242:41213 {}] regionserver.HRegionServer(976): stopping server 849dee24a242,41213,1732223653469; all regions closed. 2024-11-21T21:17:06,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741834_1010 (size=25344) 2024-11-21T21:17:06,896 DEBUG [RS:0;849dee24a242:41213 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/oldWALs 2024-11-21T21:17:06,896 INFO [RS:0;849dee24a242:41213 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 849dee24a242%2C41213%2C1732223653469.meta:.meta(num 1732223657903) 2024-11-21T21:17:06,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741833_1009 (size=16098900) 2024-11-21T21:17:06,900 DEBUG [RS:0;849dee24a242:41213 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/oldWALs 2024-11-21T21:17:06,900 INFO [RS:0;849dee24a242:41213 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 849dee24a242%2C41213%2C1732223653469:(num 1732223657010) 2024-11-21T21:17:06,900 DEBUG [RS:0;849dee24a242:41213 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T21:17:06,900 INFO [RS:0;849dee24a242:41213 {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T21:17:06,901 INFO [RS:0;849dee24a242:41213 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T21:17:06,901 INFO [RS:0;849dee24a242:41213 {}] hbase.ChoreService(370): Chore service for: regionserver/849dee24a242:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-21T21:17:06,901 INFO [RS:0;849dee24a242:41213 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T21:17:06,901 INFO [regionserver/849dee24a242:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T21:17:06,902 INFO [RS:0;849dee24a242:41213 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41213 2024-11-21T21:17:06,926 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42485-0x1015f3f4af00000, quorum=127.0.0.1:52266, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T21:17:06,926 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41213-0x1015f3f4af00001, quorum=127.0.0.1:52266, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/849dee24a242,41213,1732223653469 2024-11-21T21:17:06,926 INFO [RS:0;849dee24a242:41213 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T21:17:07,035 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [849dee24a242,41213,1732223653469] 2024-11-21T21:17:07,136 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41213-0x1015f3f4af00001, quorum=127.0.0.1:52266, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T21:17:07,136 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41213-0x1015f3f4af00001, quorum=127.0.0.1:52266, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T21:17:07,137 INFO [RS:0;849dee24a242:41213 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T21:17:07,137 INFO [RS:0;849dee24a242:41213 {}] regionserver.HRegionServer(1031): Exiting; stopping=849dee24a242,41213,1732223653469; zookeeper connection closed. 2024-11-21T21:17:07,138 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@690eeb73 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@690eeb73 2024-11-21T21:17:07,139 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-21T21:17:07,145 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/849dee24a242,41213,1732223653469 already deleted, retry=false 2024-11-21T21:17:07,145 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 849dee24a242,41213,1732223653469 expired; onlineServers=0 2024-11-21T21:17:07,145 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '849dee24a242,42485,1732223652547' ***** 2024-11-21T21:17:07,145 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-21T21:17:07,145 INFO [M:0;849dee24a242:42485 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T21:17:07,146 INFO [M:0;849dee24a242:42485 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T21:17:07,146 DEBUG [M:0;849dee24a242:42485 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-21T21:17:07,146 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-21T21:17:07,146 DEBUG [M:0;849dee24a242:42485 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-21T21:17:07,146 DEBUG [master/849dee24a242:0:becomeActiveMaster-HFileCleaner.small.0-1732223656590 {}] cleaner.HFileCleaner(306): Exit Thread[master/849dee24a242:0:becomeActiveMaster-HFileCleaner.small.0-1732223656590,5,FailOnTimeoutGroup] 2024-11-21T21:17:07,146 DEBUG [master/849dee24a242:0:becomeActiveMaster-HFileCleaner.large.0-1732223656590 {}] cleaner.HFileCleaner(306): Exit Thread[master/849dee24a242:0:becomeActiveMaster-HFileCleaner.large.0-1732223656590,5,FailOnTimeoutGroup] 2024-11-21T21:17:07,146 INFO [M:0;849dee24a242:42485 {}] hbase.ChoreService(370): Chore service for: master/849dee24a242:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-21T21:17:07,146 INFO [M:0;849dee24a242:42485 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T21:17:07,146 DEBUG [M:0;849dee24a242:42485 {}] master.HMaster(1795): Stopping service threads 2024-11-21T21:17:07,146 INFO [M:0;849dee24a242:42485 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-21T21:17:07,147 INFO [M:0;849dee24a242:42485 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T21:17:07,147 ERROR [M:0;849dee24a242:42485 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-5,5,PEWorkerGroup] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:43233 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:43233,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-21T21:17:07,147 INFO [M:0;849dee24a242:42485 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T21:17:07,147 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-21T21:17:07,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42485-0x1015f3f4af00000, quorum=127.0.0.1:52266, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-21T21:17:07,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42485-0x1015f3f4af00000, quorum=127.0.0.1:52266, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T21:17:07,220 DEBUG [M:0;849dee24a242:42485 {}] zookeeper.ZKUtil(347): master:42485-0x1015f3f4af00000, quorum=127.0.0.1:52266, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-21T21:17:07,220 WARN [M:0;849dee24a242:42485 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-21T21:17:07,221 INFO [M:0;849dee24a242:42485 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/.lastflushedseqids 2024-11-21T21:17:07,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742470_1646 (size=72) 2024-11-21T21:17:07,629 INFO [M:0;849dee24a242:42485 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T21:17:07,630 INFO [M:0;849dee24a242:42485 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-21T21:17:07,630 DEBUG [M:0;849dee24a242:42485 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-21T21:17:07,630 INFO [M:0;849dee24a242:42485 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T21:17:07,630 DEBUG [M:0;849dee24a242:42485 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T21:17:07,630 DEBUG [M:0;849dee24a242:42485 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-21T21:17:07,630 DEBUG [M:0;849dee24a242:42485 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T21:17:07,630 INFO [M:0;849dee24a242:42485 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=774.32 KB heapSize=948.06 KB 2024-11-21T21:17:07,651 DEBUG [M:0;849dee24a242:42485 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e4c6f26b1d994cd585620a5f0b1411fd is 82, key is hbase:meta,,1/info:regioninfo/1732223657988/Put/seqid=0 2024-11-21T21:17:07,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742471_1647 (size=5672) 2024-11-21T21:17:08,054 INFO [M:0;849dee24a242:42485 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2134 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e4c6f26b1d994cd585620a5f0b1411fd 2024-11-21T21:17:08,075 DEBUG [M:0;849dee24a242:42485 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ef3acc4b8ea14f4bafc80ac3ce2150c8 is 2479, key is \x00\x00\x00\x00\x00\x00\x00#/proc:d/1732223688171/Put/seqid=0 2024-11-21T21:17:08,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742472_1648 (size=47098) 2024-11-21T21:17:08,479 INFO [M:0;849dee24a242:42485 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=773.76 KB at sequenceid=2134 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ef3acc4b8ea14f4bafc80ac3ce2150c8 2024-11-21T21:17:08,482 INFO [M:0;849dee24a242:42485 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ef3acc4b8ea14f4bafc80ac3ce2150c8 2024-11-21T21:17:08,499 DEBUG [M:0;849dee24a242:42485 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/07a57b4180d54a46a3742f1761917261 is 69, key is 849dee24a242,41213,1732223653469/rs:state/1732223656734/Put/seqid=0 2024-11-21T21:17:08,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073742473_1649 (size=5156) 2024-11-21T21:17:08,502 INFO [M:0;849dee24a242:42485 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2134 (bloomFilter=true), to=hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/07a57b4180d54a46a3742f1761917261 2024-11-21T21:17:08,506 DEBUG [M:0;849dee24a242:42485 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e4c6f26b1d994cd585620a5f0b1411fd as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e4c6f26b1d994cd585620a5f0b1411fd 2024-11-21T21:17:08,508 INFO [M:0;849dee24a242:42485 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e4c6f26b1d994cd585620a5f0b1411fd, entries=8, sequenceid=2134, filesize=5.5 K 2024-11-21T21:17:08,509 DEBUG [M:0;849dee24a242:42485 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ef3acc4b8ea14f4bafc80ac3ce2150c8 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ef3acc4b8ea14f4bafc80ac3ce2150c8 2024-11-21T21:17:08,512 INFO [M:0;849dee24a242:42485 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ef3acc4b8ea14f4bafc80ac3ce2150c8 2024-11-21T21:17:08,512 INFO [M:0;849dee24a242:42485 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ef3acc4b8ea14f4bafc80ac3ce2150c8, entries=180, sequenceid=2134, filesize=46.0 K 2024-11-21T21:17:08,513 DEBUG [M:0;849dee24a242:42485 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/07a57b4180d54a46a3742f1761917261 as hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/07a57b4180d54a46a3742f1761917261 2024-11-21T21:17:08,516 INFO [M:0;849dee24a242:42485 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43233/user/jenkins/test-data/a684a5bc-0adc-81d7-5fa0-7fc8fc74286f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/07a57b4180d54a46a3742f1761917261, entries=1, sequenceid=2134, filesize=5.0 K 2024-11-21T21:17:08,517 INFO [M:0;849dee24a242:42485 {}] regionserver.HRegion(3140): Finished flush of dataSize ~774.32 KB/792901, heapSize ~947.77 KB/970512, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 887ms, sequenceid=2134, compaction requested=false 2024-11-21T21:17:08,525 INFO [M:0;849dee24a242:42485 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T21:17:08,525 DEBUG [M:0;849dee24a242:42485 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732223827630Disabling compacts and flushes for region at 1732223827630Disabling writes for close at 1732223827630Obtaining lock to block concurrent updates at 1732223827630Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732223827630Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=792901, getHeapSize=970752, getOffHeapSize=0, getCellsCount=2339 at 1732223827631 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732223827632 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732223827632Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732223827651 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732223827651Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732223828058 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732223828075 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732223828075Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732223828482 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732223828499 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732223828499Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@512787fd: reopening flushed file at 1732223828505 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c791c9c: reopening flushed file at 1732223828509 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a5e27b1: reopening flushed file at 1732223828512 (+3 ms)Finished flush of dataSize ~774.32 KB/792901, heapSize ~947.77 KB/970512, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 887ms, sequenceid=2134, compaction requested=false at 1732223828517 (+5 ms)Writing region close event to WAL at 1732223828524 (+7 ms)Closed at 1732223828524 2024-11-21T21:17:08,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35127 is added to blk_1073741830_1006 (size=932850) 2024-11-21T21:17:08,528 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T21:17:08,528 INFO [M:0;849dee24a242:42485 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-21T21:17:08,528 INFO [M:0;849dee24a242:42485 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42485 2024-11-21T21:17:08,528 INFO [M:0;849dee24a242:42485 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T21:17:08,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42485-0x1015f3f4af00000, quorum=127.0.0.1:52266, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T21:17:08,637 INFO [M:0;849dee24a242:42485 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T21:17:08,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42485-0x1015f3f4af00000, quorum=127.0.0.1:52266, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T21:17:08,644 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-21T21:17:08,648 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-21T21:17:08,648 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-21T21:17:08,648 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-21T21:17:08,649 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51952d30-d1ef-b0b3-2459-5ab0f09ed1d1/hadoop.log.dir/,STOPPED} 2024-11-21T21:17:08,652 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-21T21:17:08,652 WARN [BP-1102385231-172.17.0.2-1732223648686 heartbeating to localhost/127.0.0.1:43233 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-21T21:17:08,652 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-21T21:17:08,652 WARN [BP-1102385231-172.17.0.2-1732223648686 heartbeating to localhost/127.0.0.1:43233 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1102385231-172.17.0.2-1732223648686 (Datanode Uuid 6f411320-8877-4cf1-8745-6c5208a97a0a) service to localhost/127.0.0.1:43233 2024-11-21T21:17:08,655 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51952d30-d1ef-b0b3-2459-5ab0f09ed1d1/cluster_d7e18f9d-61ff-69fb-e450-b3a2cf11ddfa/data/data1/current/BP-1102385231-172.17.0.2-1732223648686 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-21T21:17:08,655 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51952d30-d1ef-b0b3-2459-5ab0f09ed1d1/cluster_d7e18f9d-61ff-69fb-e450-b3a2cf11ddfa/data/data2/current/BP-1102385231-172.17.0.2-1732223648686 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-21T21:17:08,655 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-21T21:17:08,661 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-21T21:17:08,662 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-21T21:17:08,662 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-21T21:17:08,662 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-21T21:17:08,662 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/51952d30-d1ef-b0b3-2459-5ab0f09ed1d1/hadoop.log.dir/,STOPPED} 2024-11-21T21:17:08,678 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-21T21:17:08,798 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down